{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "09f7126f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c6ab2b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gc\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import time\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "import category_encoders as ce\n",
    "import networkx as nx\n",
    "import pickle\n",
    "import lightgbm as lgb\n",
    "import catboost as cat\n",
    "import xgboost as xgb\n",
    "import seaborn as sns\n",
    "from datetime import timedelta\n",
    "from gensim.models import Word2Vec\n",
    "from io import StringIO\n",
    "from tqdm import tqdm\n",
    "from lightgbm import LGBMClassifier\n",
    "from lightgbm import log_evaluation, early_stopping\n",
    "from sklearn.metrics import roc_curve\n",
    "from scipy.stats import chi2_contingency, pearsonr\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder, LabelEncoder\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.feature_extraction import FeatureHasher\n",
    "from sklearn.model_selection import StratifiedKFold, KFold, train_test_split, GridSearchCV\n",
    "from category_encoders import TargetEncoder\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from autogluon.tabular import TabularDataset, TabularPredictor, FeatureMetadata\n",
    "from autogluon.features.generators import AsTypeFeatureGenerator, BulkFeatureGenerator, DropUniqueFeatureGenerator, FillNaFeatureGenerator, PipelineFeatureGenerator\n",
    "from autogluon.features.generators import CategoryFeatureGenerator, IdentityFeatureGenerator, AutoMLPipelineFeatureGenerator\n",
    "from autogluon.common.features.types import R_INT, R_FLOAT\n",
    "from autogluon.core.metrics import make_scorer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a08d6044",
   "metadata": {},
   "source": [
    "## 数据导入"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df055add",
   "metadata": {},
   "source": [
    "## 通用导入函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "74bcbf7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data_from_directory(directory):\n",
    "    \"\"\"\n",
    "    遍历目录加载所有CSV文件，将其作为独立的DataFrame变量\n",
    "\n",
    "    参数:\n",
    "    - directory: 输入的数据路径\n",
    "    \n",
    "    返回:\n",
    "    - 含有数据集名称的列表\n",
    "    \"\"\"\n",
    "    dataset_names = []\n",
    "    for filename in os.listdir(directory):\n",
    "        if filename.endswith(\".csv\"):\n",
    "            dataset_name = os.path.splitext(filename)[0] + '_data' # 获取文件名作为变量名\n",
    "            file_path = os.path.join(directory, filename)  # 完整的文件路径\n",
    "            globals()[dataset_name] = pd.read_csv(file_path)  # 将文件加载为DataFrame并赋值给全局变量\n",
    "            dataset_names.append(dataset_name)\n",
    "            print(f\"数据集 {dataset_name} 已加载为 DataFrame\")\n",
    "\n",
    "    return dataset_names"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "993c86ce",
   "metadata": {},
   "source": [
    "## 训练集导入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "722d1e40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "数据集 TRAIN_ASSET_data 已加载为 DataFrame\n",
      "数据集 TRAIN_CCD_TR_DTL_data 已加载为 DataFrame\n",
      "数据集 TRAIN_MB_CUST_INFO_data 已加载为 DataFrame\n",
      "数据集 TRAIN_MB_PAGEVIEW_DTL_data 已加载为 DataFrame\n",
      "数据集 TRAIN_MB_TRNFLW_DTL_data 已加载为 DataFrame\n",
      "数据集 TRAIN_NATURE_data 已加载为 DataFrame\n",
      "数据集 TRAIN_PROD_HOLD_data 已加载为 DataFrame\n",
      "数据集 TRAIN_TR_APS_DTL_data 已加载为 DataFrame\n"
     ]
    }
   ],
   "source": [
    "train_load_dt = './data/Train'\n",
    "train_data_name = load_data_from_directory(train_load_dt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82e70968",
   "metadata": {},
   "source": [
    "## 测试集导入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c3ef0f15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "数据集 A_ASSET_data 已加载为 DataFrame\n",
      "数据集 A_CCD_TR_DTL_data 已加载为 DataFrame\n",
      "数据集 A_MB_CUST_INFO_data 已加载为 DataFrame\n",
      "数据集 A_MB_PAGEVIEW_DTL_data 已加载为 DataFrame\n",
      "数据集 A_MB_TRNFLW_DTL_data 已加载为 DataFrame\n",
      "数据集 A_PROD_HOLD_data 已加载为 DataFrame\n",
      "数据集 A_TEST_NATURE_data 已加载为 DataFrame\n",
      "数据集 A_TR_APS_DTL_data 已加载为 DataFrame\n"
     ]
    }
   ],
   "source": [
    "A_load_dt = './data/A'\n",
    "A_data_name = load_data_from_directory(A_load_dt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6a9129f",
   "metadata": {},
   "source": [
    "# 特征工程"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3632eae8",
   "metadata": {},
   "source": [
    "## 掌银金融性交易流水表特征工程"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d211d2d",
   "metadata": {},
   "source": [
    "### 数据预处理与时间特征生成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d2bd4f21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN - 参考日期(最大日期): 2025-03-10 00:00:00\n",
      "TRAIN - 数据预处理完成! 数据形状: (2000, 19)\n",
      "TRAIN - 日期范围: 2025-03-07 00:00:00 至 2025-03-10 00:00:00\n",
      "TRAIN - 客户数: 1291\n",
      "TRAIN - 交易总数: 2000\n",
      "TRAIN - 业务代码数: 43\n",
      "TRAIN - 状态种类数: 3\n",
      "TRAIN - 代收付类型数: 8\n",
      "TRAIN - 客户类别数: 3\n",
      "TEST - 参考日期(最大日期): 2025-06-05 00:00:00\n",
      "TEST - 数据预处理完成! 数据形状: (1000, 19)\n",
      "TEST - 日期范围: 2025-04-16 00:00:00 至 2025-06-05 00:00:00\n",
      "TEST - 客户数: 457\n",
      "TEST - 交易总数: 1000\n",
      "TEST - 业务代码数: 21\n",
      "TEST - 状态种类数: 3\n",
      "TEST - 代收付类型数: 6\n",
      "TEST - 客户类别数: 2\n"
     ]
    }
   ],
   "source": [
    "def preprocess_mb_trnflw_data(df, stage='train'):\n",
    "    \"\"\"\n",
    "    掌银金融性交易流水表数据预处理\n",
    "    \n",
    "    参数:\n",
    "    - df: DataFrame\n",
    "    - stage: 'train' 或 'test'\n",
    "    \n",
    "    返回:\n",
    "    - 预处理后的DataFrame和参考日期\n",
    "    \"\"\"\n",
    "    df = df.copy()\n",
    "    \n",
    "    # 转换日期格式\n",
    "    df['DATE'] = pd.to_datetime(df['DATE'], format='%Y%m%d')\n",
    "    \n",
    "    # 获取参考日期(最大日期)\n",
    "    reference_date = df['DATE'].max()\n",
    "    print(f\"{stage} - 参考日期(最大日期): {reference_date}\")\n",
    "    \n",
    "    # 时间特征\n",
    "    df['year'] = df['DATE'].dt.year\n",
    "    df['month'] = df['DATE'].dt.month\n",
    "    df['day'] = df['DATE'].dt.day\n",
    "    df['weekday'] = df['DATE'].dt.dayofweek  # 0=周一, 6=周日\n",
    "    df['week'] = df['DATE'].dt.isocalendar().week\n",
    "    df['is_weekend'] = df['weekday'].apply(lambda x: 1 if x >= 5 else 0)\n",
    "    df['is_month_start'] = df['DATE'].dt.is_month_start.astype(int)\n",
    "    df['is_month_end'] = df['DATE'].dt.is_month_end.astype(int)\n",
    "    df['day_of_month'] = df['DATE'].dt.day\n",
    "    df['days_from_ref'] = (reference_date - df['DATE']).dt.days\n",
    "    \n",
    "    # 处理缺失值\n",
    "    df['TFT_TRNAMT'] = df['TFT_TRNAMT'].fillna(0)\n",
    "    df['TFT_STT'] = df['TFT_STT'].fillna('UNKNOWN')\n",
    "    df['TFT_PAYCOD'] = df['TFT_PAYCOD'].fillna('UNKNOWN')\n",
    "    df['TFT_CSTTYPE'] = df['TFT_CSTTYPE'].fillna(-1)\n",
    "    \n",
    "    # 金额对数变换(避免极端值影响)\n",
    "    df['TFT_TRNAMT_log'] = np.log1p(df['TFT_TRNAMT'])\n",
    "    \n",
    "    # 金额分档\n",
    "    df['amount_bin'] = pd.cut(df['TFT_TRNAMT'], \n",
    "                               bins=[0, 100, 500, 1000, 5000, 10000, 50000, 100000, float('inf')],\n",
    "                               labels=['0_100', '100_500', '500_1k', '1k_5k', '5k_10k', '10k_50k', '50k_100k', '100k_plus'])\n",
    "    \n",
    "    print(f\"{stage} - 数据预处理完成! 数据形状: {df.shape}\")\n",
    "    print(f\"{stage} - 日期范围: {df['DATE'].min()} 至 {df['DATE'].max()}\")\n",
    "    print(f\"{stage} - 客户数: {df['CUST_NO'].nunique()}\")\n",
    "    print(f\"{stage} - 交易总数: {len(df)}\")\n",
    "    print(f\"{stage} - 业务代码数: {df['TFT_STDBSNCOD'].nunique()}\")\n",
    "    print(f\"{stage} - 状态种类数: {df['TFT_STT'].nunique()}\")\n",
    "    print(f\"{stage} - 代收付类型数: {df['TFT_PAYCOD'].nunique()}\")\n",
    "    print(f\"{stage} - 客户类别数: {df['TFT_CSTTYPE'].nunique()}\")\n",
    "    \n",
    "    return df, reference_date\n",
    "\n",
    "# 预处理训练集和测试集\n",
    "train_df, train_ref_date = preprocess_mb_trnflw_data(TRAIN_MB_TRNFLW_DTL_data, stage='TRAIN')\n",
    "test_df, test_ref_date = preprocess_mb_trnflw_data(A_MB_TRNFLW_DTL_data, stage='TEST')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e0d071e",
   "metadata": {},
   "source": [
    "### 特征工程主函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9fe6e40a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_mb_trnflw_features(df, reference_date, prefix='mb_trnflw'):\n",
    "    \"\"\"\n",
    "    创建掌银金融性交易流水表的全面特征\n",
    "    \n",
    "    参数:\n",
    "    - df: 预处理后的掌银金融性交易流水表DataFrame\n",
    "    - reference_date: 参考日期\n",
    "    - prefix: 特征前缀\n",
    "    \n",
    "    返回:\n",
    "    - 特征DataFrame\n",
    "    \"\"\"\n",
    "    print(f\"开始构建{prefix}特征...\")\n",
    "    print(\"=\" * 80)\n",
    "    \n",
    "    features_list = []\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 1. 基础统计特征\n",
    "    # ====================================================================================\n",
    "    print(\"1. 基础统计特征...\")\n",
    "    \n",
    "    basic_stats = df.groupby('CUST_NO').agg({\n",
    "        'TFT_TRNAMT': ['count', 'sum', 'mean', 'median', 'std', 'min', 'max', \n",
    "                       lambda x: x.quantile(0.25), lambda x: x.quantile(0.75), \n",
    "                       lambda x: x.quantile(0.1), lambda x: x.quantile(0.9)],\n",
    "        'TFT_TRNAMT_log': ['mean', 'std', 'min', 'max'],\n",
    "        'TFT_STDBSNCOD': 'nunique',\n",
    "        'TFT_STT': 'nunique',\n",
    "        'TFT_PAYCOD': 'nunique',\n",
    "        'TFT_CSTTYPE': 'nunique',\n",
    "        'DATE': ['min', 'max', 'nunique'],\n",
    "    }).reset_index()\n",
    "    \n",
    "    basic_stats.columns = ['CUST_NO',\n",
    "                           f'{prefix}_count', f'{prefix}_amt_sum', f'{prefix}_amt_mean',\n",
    "                           f'{prefix}_amt_median', f'{prefix}_amt_std', f'{prefix}_amt_min',\n",
    "                           f'{prefix}_amt_max', f'{prefix}_amt_q25', f'{prefix}_amt_q75',\n",
    "                           f'{prefix}_amt_q10', f'{prefix}_amt_q90',\n",
    "                           f'{prefix}_amt_log_mean', f'{prefix}_amt_log_std', \n",
    "                           f'{prefix}_amt_log_min', f'{prefix}_amt_log_max',\n",
    "                           f'{prefix}_bsncod_nunique', f'{prefix}_stt_nunique',\n",
    "                           f'{prefix}_paycod_nunique', f'{prefix}_csttype_nunique',\n",
    "                           f'{prefix}_first_date', f'{prefix}_last_date', f'{prefix}_date_nunique']\n",
    "    \n",
    "    # 派生特征\n",
    "    basic_stats[f'{prefix}_amt_range'] = basic_stats[f'{prefix}_amt_max'] - basic_stats[f'{prefix}_amt_min']\n",
    "    basic_stats[f'{prefix}_amt_iqr'] = basic_stats[f'{prefix}_amt_q75'] - basic_stats[f'{prefix}_amt_q25']\n",
    "    basic_stats[f'{prefix}_amt_cv'] = basic_stats[f'{prefix}_amt_std'] / (basic_stats[f'{prefix}_amt_mean'] + 1e-5)\n",
    "    basic_stats[f'{prefix}_amt_log_range'] = basic_stats[f'{prefix}_amt_log_max'] - basic_stats[f'{prefix}_amt_log_min']\n",
    "    basic_stats[f'{prefix}_days_since_first'] = (reference_date - basic_stats[f'{prefix}_first_date']).dt.days\n",
    "    basic_stats[f'{prefix}_days_since_last'] = (reference_date - basic_stats[f'{prefix}_last_date']).dt.days\n",
    "    basic_stats[f'{prefix}_active_span'] = (basic_stats[f'{prefix}_last_date'] - basic_stats[f'{prefix}_first_date']).dt.days + 1\n",
    "    basic_stats[f'{prefix}_freq_per_day'] = basic_stats[f'{prefix}_count'] / (basic_stats[f'{prefix}_active_span'] + 1)\n",
    "    basic_stats[f'{prefix}_active_rate'] = basic_stats[f'{prefix}_date_nunique'] / (basic_stats[f'{prefix}_active_span'] + 1)\n",
    "    basic_stats[f'{prefix}_avg_trn_per_active_day'] = basic_stats[f'{prefix}_count'] / (basic_stats[f'{prefix}_date_nunique'] + 1)\n",
    "    basic_stats[f'{prefix}_amt_per_day'] = basic_stats[f'{prefix}_amt_sum'] / (basic_stats[f'{prefix}_active_span'] + 1)\n",
    "    \n",
    "    # 多样性指标\n",
    "    basic_stats[f'{prefix}_diversity_score'] = (basic_stats[f'{prefix}_bsncod_nunique'] * \n",
    "                                                 basic_stats[f'{prefix}_stt_nunique'] * \n",
    "                                                 basic_stats[f'{prefix}_paycod_nunique'])\n",
    "    \n",
    "    basic_stats = basic_stats.drop([f'{prefix}_first_date', f'{prefix}_last_date'], axis=1)\n",
    "    features_list.append(basic_stats)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 2. 时间窗口特征(滑动窗口)\n",
    "    # ====================================================================================\n",
    "    print(\"2. 时间窗口特征...\")\n",
    "    \n",
    "    time_windows = [1, 3, 7, 14, 30, 60, 90]\n",
    "    for window in time_windows:\n",
    "        df_window = df[df['days_from_ref'] < window]\n",
    "        \n",
    "        if len(df_window) > 0:\n",
    "            window_stats = df_window.groupby('CUST_NO').agg({\n",
    "                'TFT_TRNAMT': ['count', 'sum', 'mean', 'max', 'std'],\n",
    "                'TFT_STDBSNCOD': 'nunique',\n",
    "                'TFT_STT': 'nunique',\n",
    "                'DATE': 'nunique'\n",
    "            }).reset_index()\n",
    "            \n",
    "            window_stats.columns = ['CUST_NO',\n",
    "                                    f'{prefix}_count_last{window}d',\n",
    "                                    f'{prefix}_amt_sum_last{window}d',\n",
    "                                    f'{prefix}_amt_mean_last{window}d',\n",
    "                                    f'{prefix}_amt_max_last{window}d',\n",
    "                                    f'{prefix}_amt_std_last{window}d',\n",
    "                                    f'{prefix}_bsncod_nunique_last{window}d',\n",
    "                                    f'{prefix}_stt_nunique_last{window}d',\n",
    "                                    f'{prefix}_active_days_last{window}d']\n",
    "            \n",
    "            window_stats[f'{prefix}_freq_last{window}d'] = window_stats[f'{prefix}_count_last{window}d'] / window\n",
    "            window_stats[f'{prefix}_active_rate_last{window}d'] = window_stats[f'{prefix}_active_days_last{window}d'] / window\n",
    "            window_stats[f'{prefix}_amt_per_day_last{window}d'] = window_stats[f'{prefix}_amt_sum_last{window}d'] / window\n",
    "            \n",
    "            features_list.append(window_stats)\n",
    "    \n",
    "    # 时间窗口趋势特征\n",
    "    temp_df = features_list[0].copy()\n",
    "    for i in range(1, len(features_list)):\n",
    "        temp_df = temp_df.merge(features_list[i], on='CUST_NO', how='left')\n",
    "    \n",
    "    # 短期vs长期趋势\n",
    "    if f'{prefix}_count_last7d' in temp_df.columns and f'{prefix}_count_last30d' in temp_df.columns:\n",
    "        temp_df[f'{prefix}_trend_7d_30d'] = temp_df[f'{prefix}_count_last7d'] / (temp_df[f'{prefix}_count_last30d'] / 30 * 7 + 1e-5)\n",
    "        temp_df[f'{prefix}_amt_trend_7d_30d'] = temp_df[f'{prefix}_amt_sum_last7d'] / (temp_df[f'{prefix}_amt_sum_last30d'] / 30 * 7 + 1e-5)\n",
    "    \n",
    "    if f'{prefix}_count_last14d' in temp_df.columns and f'{prefix}_count_last60d' in temp_df.columns:\n",
    "        temp_df[f'{prefix}_trend_14d_60d'] = temp_df[f'{prefix}_count_last14d'] / (temp_df[f'{prefix}_count_last60d'] / 60 * 14 + 1e-5)\n",
    "    \n",
    "    if f'{prefix}_count_last30d' in temp_df.columns and f'{prefix}_count_last90d' in temp_df.columns:\n",
    "        temp_df[f'{prefix}_trend_30d_90d'] = temp_df[f'{prefix}_count_last30d'] / (temp_df[f'{prefix}_count_last90d'] / 90 * 30 + 1e-5)\n",
    "        temp_df[f'{prefix}_amt_trend_30d_90d'] = temp_df[f'{prefix}_amt_sum_last30d'] / (temp_df[f'{prefix}_amt_sum_last90d'] / 90 * 30 + 1e-5)\n",
    "    \n",
    "    # 活跃度趋势\n",
    "    if f'{prefix}_active_rate_last7d' in temp_df.columns and f'{prefix}_active_rate_last30d' in temp_df.columns:\n",
    "        temp_df[f'{prefix}_active_trend_7d_30d'] = temp_df[f'{prefix}_active_rate_last7d'] / (temp_df[f'{prefix}_active_rate_last30d'] + 1e-5)\n",
    "    \n",
    "    features_list = [temp_df]\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 3. 周末vs工作日特征\n",
    "    # ====================================================================================\n",
    "    print(\"3. 周末vs工作日特征...\")\n",
    "    \n",
    "    # 周末\n",
    "    df_weekend = df[df['is_weekend'] == 1]\n",
    "    if len(df_weekend) > 0:\n",
    "        weekend_stats = df_weekend.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum', 'mean', 'max'],\n",
    "            'TFT_STDBSNCOD': 'nunique',\n",
    "            'DATE': 'nunique'\n",
    "        }).reset_index()\n",
    "        weekend_stats.columns = ['CUST_NO', f'{prefix}_weekend_count', f'{prefix}_weekend_amt_sum',\n",
    "                                 f'{prefix}_weekend_amt_mean', f'{prefix}_weekend_amt_max',\n",
    "                                 f'{prefix}_weekend_bsncod_nunique', f'{prefix}_weekend_active_days']\n",
    "        features_list.append(weekend_stats)\n",
    "    \n",
    "    # 工作日\n",
    "    df_weekday = df[df['is_weekend'] == 0]\n",
    "    if len(df_weekday) > 0:\n",
    "        weekday_stats = df_weekday.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum', 'mean', 'max'],\n",
    "            'TFT_STDBSNCOD': 'nunique',\n",
    "            'DATE': 'nunique'\n",
    "        }).reset_index()\n",
    "        weekday_stats.columns = ['CUST_NO', f'{prefix}_weekday_count', f'{prefix}_weekday_amt_sum',\n",
    "                                 f'{prefix}_weekday_amt_mean', f'{prefix}_weekday_amt_max',\n",
    "                                 f'{prefix}_weekday_bsncod_nunique', f'{prefix}_weekday_active_days']\n",
    "        features_list.append(weekday_stats)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 4. 标准业务代码(TFT_STDBSNCOD)特征\n",
    "    # ====================================================================================\n",
    "    print(\"4. 标准业务代码特征...\")\n",
    "    \n",
    "    # Top20业务代码\n",
    "    top_bsncods = df['TFT_STDBSNCOD'].value_counts().head(20).index.tolist()\n",
    "    for idx, bsncod in enumerate(top_bsncods):\n",
    "        df_bsncod = df[df['TFT_STDBSNCOD'] == bsncod]\n",
    "        bsncod_stats = df_bsncod.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum', 'mean', 'max']\n",
    "        }).reset_index()\n",
    "        bsncod_stats.columns = ['CUST_NO',\n",
    "                               f'{prefix}_bsncod_{idx+1}_count',\n",
    "                               f'{prefix}_bsncod_{idx+1}_amt_sum',\n",
    "                               f'{prefix}_bsncod_{idx+1}_amt_mean',\n",
    "                               f'{prefix}_bsncod_{idx+1}_amt_max']\n",
    "        features_list.append(bsncod_stats)\n",
    "    \n",
    "    # 业务代码频率分布\n",
    "    bsncod_freq = df.groupby(['CUST_NO', 'TFT_STDBSNCOD']).size().reset_index(name='freq')\n",
    "    \n",
    "    # 业务代码熵(多样性)\n",
    "    bsncod_entropy = bsncod_freq.groupby('CUST_NO').apply(\n",
    "        lambda x: -np.sum((x['freq'] / x['freq'].sum()) * np.log(x['freq'] / x['freq'].sum() + 1e-10))\n",
    "    ).reset_index(name=f'{prefix}_bsncod_entropy')\n",
    "    features_list.append(bsncod_entropy)\n",
    "    \n",
    "    # 最常用业务代码占比\n",
    "    bsncod_max = bsncod_freq.groupby('CUST_NO')['freq'].max().reset_index(name=f'{prefix}_bsncod_max_freq')\n",
    "    bsncod_total = bsncod_freq.groupby('CUST_NO')['freq'].sum().reset_index(name=f'{prefix}_bsncod_total_freq')\n",
    "    bsncod_concentration = bsncod_max.merge(bsncod_total, on='CUST_NO')\n",
    "    bsncod_concentration[f'{prefix}_bsncod_concentration'] = (bsncod_concentration[f'{prefix}_bsncod_max_freq'] / \n",
    "                                                              bsncod_concentration[f'{prefix}_bsncod_total_freq'])\n",
    "    features_list.append(bsncod_concentration[['CUST_NO', f'{prefix}_bsncod_concentration']])\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 5. 状态(TFT_STT)特征\n",
    "    # ====================================================================================\n",
    "    print(\"5. 状态特征...\")\n",
    "    \n",
    "    # Top10状态\n",
    "    top_stts = df['TFT_STT'].value_counts().head(10).index.tolist()\n",
    "    for idx, stt in enumerate(top_stts):\n",
    "        df_stt = df[df['TFT_STT'] == stt]\n",
    "        stt_stats = df_stt.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum', 'mean']\n",
    "        }).reset_index()\n",
    "        stt_stats.columns = ['CUST_NO',\n",
    "                            f'{prefix}_stt_{idx+1}_count',\n",
    "                            f'{prefix}_stt_{idx+1}_amt_sum',\n",
    "                            f'{prefix}_stt_{idx+1}_amt_mean']\n",
    "        features_list.append(stt_stats)\n",
    "    \n",
    "    # 状态成功率(假设'T'为成功状态)\n",
    "    if 'T' in df['TFT_STT'].values:\n",
    "        df_success = df[df['TFT_STT'] == 'T']\n",
    "        if len(df_success) > 0:\n",
    "            success_count = df_success.groupby('CUST_NO').size().reset_index(name=f'{prefix}_success_count')\n",
    "            features_list.append(success_count)\n",
    "    \n",
    "    # 状态多样性\n",
    "    stt_diversity = df.groupby('CUST_NO')['TFT_STT'].nunique().reset_index(name=f'{prefix}_stt_diversity')\n",
    "    features_list.append(stt_diversity)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 6. 代收付类型(TFT_PAYCOD)特征\n",
    "    # ====================================================================================\n",
    "    print(\"6. 代收付类型特征...\")\n",
    "    \n",
    "    # Top10代收付类型\n",
    "    top_paycods = df['TFT_PAYCOD'].value_counts().head(10).index.tolist()\n",
    "    for idx, paycod in enumerate(top_paycods):\n",
    "        df_paycod = df[df['TFT_PAYCOD'] == paycod]\n",
    "        paycod_stats = df_paycod.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum', 'mean', 'max']\n",
    "        }).reset_index()\n",
    "        paycod_stats.columns = ['CUST_NO',\n",
    "                               f'{prefix}_paycod_{idx+1}_count',\n",
    "                               f'{prefix}_paycod_{idx+1}_amt_sum',\n",
    "                               f'{prefix}_paycod_{idx+1}_amt_mean',\n",
    "                               f'{prefix}_paycod_{idx+1}_amt_max']\n",
    "        features_list.append(paycod_stats)\n",
    "    \n",
    "    # 代收付类型熵\n",
    "    paycod_freq = df.groupby(['CUST_NO', 'TFT_PAYCOD']).size().reset_index(name='freq')\n",
    "    paycod_entropy = paycod_freq.groupby('CUST_NO').apply(\n",
    "        lambda x: -np.sum((x['freq'] / x['freq'].sum()) * np.log(x['freq'] / x['freq'].sum() + 1e-10))\n",
    "    ).reset_index(name=f'{prefix}_paycod_entropy')\n",
    "    features_list.append(paycod_entropy)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 7. 客户类别(TFT_CSTTYPE)特征\n",
    "    # ====================================================================================\n",
    "    print(\"7. 客户类别特征...\")\n",
    "    \n",
    "    # 各客户类别统计\n",
    "    for csttype in df['TFT_CSTTYPE'].dropna().unique():\n",
    "        if csttype != -1:\n",
    "            df_csttype = df[df['TFT_CSTTYPE'] == csttype]\n",
    "            if len(df_csttype) > 0:\n",
    "                csttype_stats = df_csttype.groupby('CUST_NO').agg({\n",
    "                    'TFT_TRNAMT': ['count', 'sum', 'mean']\n",
    "                }).reset_index()\n",
    "                csttype_stats.columns = ['CUST_NO',\n",
    "                                        f'{prefix}_csttype_{int(csttype)}_count',\n",
    "                                        f'{prefix}_csttype_{int(csttype)}_amt_sum',\n",
    "                                        f'{prefix}_csttype_{int(csttype)}_amt_mean']\n",
    "                features_list.append(csttype_stats)\n",
    "    \n",
    "    # 客户类别主要类型(最常用)\n",
    "    csttype_main = df.groupby('CUST_NO')['TFT_CSTTYPE'].agg(lambda x: x.value_counts().index[0] if len(x) > 0 else -1).reset_index(name=f'{prefix}_csttype_main')\n",
    "    features_list.append(csttype_main)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 8. 金额分档特征\n",
    "    # ====================================================================================\n",
    "    print(\"8. 金额分档特征...\")\n",
    "    \n",
    "    amount_bins_labels = ['0_100', '100_500', '500_1k', '1k_5k', '5k_10k', '10k_50k', '50k_100k', '100k_plus']\n",
    "    \n",
    "    for label in amount_bins_labels:\n",
    "        df_bin = df[df['amount_bin'] == label]\n",
    "        if len(df_bin) > 0:\n",
    "            amt_bin_stats = df_bin.groupby('CUST_NO').agg({\n",
    "                'TFT_TRNAMT': ['count', 'sum']\n",
    "            }).reset_index()\n",
    "            amt_bin_stats.columns = ['CUST_NO', \n",
    "                                    f'{prefix}_amt_{label}_count',\n",
    "                                    f'{prefix}_amt_{label}_sum']\n",
    "            features_list.append(amt_bin_stats)\n",
    "    \n",
    "    # 大额交易(Top10%)\n",
    "    large_amt_threshold = df['TFT_TRNAMT'].quantile(0.9)\n",
    "    df_large = df[df['TFT_TRNAMT'] >= large_amt_threshold]\n",
    "    if len(df_large) > 0:\n",
    "        large_amt_stats = df_large.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum', 'mean']\n",
    "        }).reset_index()\n",
    "        large_amt_stats.columns = ['CUST_NO', f'{prefix}_large_amt_count', \n",
    "                                   f'{prefix}_large_amt_sum', f'{prefix}_large_amt_mean']\n",
    "        features_list.append(large_amt_stats)\n",
    "    \n",
    "    # 小额交易(Bottom10%)\n",
    "    small_amt_threshold = df['TFT_TRNAMT'].quantile(0.1)\n",
    "    df_small = df[df['TFT_TRNAMT'] <= small_amt_threshold]\n",
    "    if len(df_small) > 0:\n",
    "        small_amt_stats = df_small.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum']\n",
    "        }).reset_index()\n",
    "        small_amt_stats.columns = ['CUST_NO', f'{prefix}_small_amt_count', f'{prefix}_small_amt_sum']\n",
    "        features_list.append(small_amt_stats)\n",
    "    \n",
    "    print(f\"当前已生成 {len(features_list)} 组特征\")\n",
    "    \n",
    "    return features_list\n",
    "\n",
    "# 注意: 这只是第一部分,我们将在下一个cell继续添加更多特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c07103be",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_mb_trnflw_features_part2(df, reference_date, features_list, prefix='mb_trnflw'):\n",
    "    \"\"\"\n",
    "    创建掌银金融性交易流水表的高级特征(第二部分)\n",
    "    \n",
    "    参数:\n",
    "    - df: 预处理后的掌银金融性交易流水表DataFrame\n",
    "    - reference_date: 参考日期\n",
    "    - features_list: 已有特征列表\n",
    "    - prefix: 特征前缀\n",
    "    \n",
    "    返回:\n",
    "    - 特征DataFrame列表\n",
    "    \"\"\"\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 9. 时间周期特征\n",
    "    # ====================================================================================\n",
    "    print(\"9. 时间周期特征...\")\n",
    "    \n",
    "    # 按月统计\n",
    "    df_monthly = df.groupby(['CUST_NO', 'month']).agg({\n",
    "        'TFT_TRNAMT': ['count', 'sum', 'mean', 'max']\n",
    "    }).reset_index()\n",
    "    df_monthly.columns = ['CUST_NO', 'month', 'count', 'sum', 'mean', 'max']\n",
    "    \n",
    "    monthly_trend = df_monthly.groupby('CUST_NO').agg({\n",
    "        'count': ['mean', 'std', 'min', 'max'],\n",
    "        'sum': ['mean', 'std', 'max'],\n",
    "        'mean': ['mean', 'std']\n",
    "    }).reset_index()\n",
    "    monthly_trend.columns = ['CUST_NO',\n",
    "                            f'{prefix}_monthly_count_mean', f'{prefix}_monthly_count_std',\n",
    "                            f'{prefix}_monthly_count_min', f'{prefix}_monthly_count_max',\n",
    "                            f'{prefix}_monthly_amt_sum_mean', f'{prefix}_monthly_amt_sum_std',\n",
    "                            f'{prefix}_monthly_amt_sum_max',\n",
    "                            f'{prefix}_monthly_amt_mean_mean', f'{prefix}_monthly_amt_mean_std']\n",
    "    features_list.append(monthly_trend)\n",
    "    \n",
    "    # 活跃月份数\n",
    "    active_months = df.groupby('CUST_NO')['month'].nunique().reset_index(name=f'{prefix}_active_months')\n",
    "    features_list.append(active_months)\n",
    "    \n",
    "    # 按周统计\n",
    "    df_weekly = df.groupby(['CUST_NO', 'week']).agg({\n",
    "        'TFT_TRNAMT': ['count', 'sum', 'mean']\n",
    "    }).reset_index()\n",
    "    df_weekly.columns = ['CUST_NO', 'week', 'count', 'sum', 'mean']\n",
    "    \n",
    "    weekly_trend = df_weekly.groupby('CUST_NO').agg({\n",
    "        'count': ['mean', 'std', 'max'],\n",
    "        'sum': ['mean', 'std'],\n",
    "        'mean': ['mean']\n",
    "    }).reset_index()\n",
    "    weekly_trend.columns = ['CUST_NO',\n",
    "                           f'{prefix}_weekly_count_mean', f'{prefix}_weekly_count_std', f'{prefix}_weekly_count_max',\n",
    "                           f'{prefix}_weekly_amt_sum_mean', f'{prefix}_weekly_amt_sum_std',\n",
    "                           f'{prefix}_weekly_amt_mean_mean']\n",
    "    features_list.append(weekly_trend)\n",
    "    \n",
    "    # 按星期几统计\n",
    "    for day in range(7):\n",
    "        df_day = df[df['weekday'] == day]\n",
    "        if len(df_day) > 0:\n",
    "            day_stats = df_day.groupby('CUST_NO').agg({\n",
    "                'TFT_TRNAMT': ['count', 'sum', 'mean']\n",
    "            }).reset_index()\n",
    "            day_stats.columns = ['CUST_NO',\n",
    "                                f'{prefix}_weekday{day}_count',\n",
    "                                f'{prefix}_weekday{day}_amt_sum',\n",
    "                                f'{prefix}_weekday{day}_amt_mean']\n",
    "            features_list.append(day_stats)\n",
    "    \n",
    "    # 月初月末特征\n",
    "    df_month_start = df[df['is_month_start'] == 1]\n",
    "    if len(df_month_start) > 0:\n",
    "        month_start_stats = df_month_start.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum', 'mean']\n",
    "        }).reset_index()\n",
    "        month_start_stats.columns = ['CUST_NO', f'{prefix}_month_start_count', \n",
    "                                     f'{prefix}_month_start_amt_sum', f'{prefix}_month_start_amt_mean']\n",
    "        features_list.append(month_start_stats)\n",
    "    \n",
    "    df_month_end = df[df['is_month_end'] == 1]\n",
    "    if len(df_month_end) > 0:\n",
    "        month_end_stats = df_month_end.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum', 'mean']\n",
    "        }).reset_index()\n",
    "        month_end_stats.columns = ['CUST_NO', f'{prefix}_month_end_count', \n",
    "                                   f'{prefix}_month_end_amt_sum', f'{prefix}_month_end_amt_mean']\n",
    "        features_list.append(month_end_stats)\n",
    "    \n",
    "    # 月中旬(10-20号)\n",
    "    df_mid_month = df[(df['day_of_month'] >= 10) & (df['day_of_month'] <= 20)]\n",
    "    if len(df_mid_month) > 0:\n",
    "        mid_month_stats = df_mid_month.groupby('CUST_NO').agg({\n",
    "            'TFT_TRNAMT': ['count', 'sum', 'mean']\n",
    "        }).reset_index()\n",
    "        mid_month_stats.columns = ['CUST_NO', f'{prefix}_mid_month_count', \n",
    "                                   f'{prefix}_mid_month_amt_sum', f'{prefix}_mid_month_amt_mean']\n",
    "        features_list.append(mid_month_stats)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 10. 连续性与规律性特征\n",
    "    # ====================================================================================\n",
    "    print(\"10. 连续性与规律性特征...\")\n",
    "    \n",
    "    # 最大连续活跃天数\n",
    "    def calc_max_consecutive_days(group):\n",
    "        dates = sorted(group['DATE'].unique())\n",
    "        if len(dates) <= 1:\n",
    "            return len(dates)\n",
    "        max_consecutive = 1\n",
    "        current_consecutive = 1\n",
    "        for i in range(1, len(dates)):\n",
    "            if (dates[i] - dates[i-1]).days == 1:\n",
    "                current_consecutive += 1\n",
    "                max_consecutive = max(max_consecutive, current_consecutive)\n",
    "            else:\n",
    "                current_consecutive = 1\n",
    "        return max_consecutive\n",
    "    \n",
    "    consecutive_days = df.groupby('CUST_NO').apply(\n",
    "        calc_max_consecutive_days\n",
    "    ).reset_index(name=f'{prefix}_max_consecutive_days')\n",
    "    features_list.append(consecutive_days)\n",
    "    \n",
    "    # 最大连续非活跃天数\n",
    "    def calc_max_gap_days(group):\n",
    "        dates = sorted(group['DATE'].unique())\n",
    "        if len(dates) <= 1:\n",
    "            return 0\n",
    "        max_gap = 0\n",
    "        for i in range(1, len(dates)):\n",
    "            gap = (dates[i] - dates[i-1]).days - 1\n",
    "            max_gap = max(max_gap, gap)\n",
    "        return max_gap\n",
    "    \n",
    "    gap_days = df.groupby('CUST_NO').apply(\n",
    "        calc_max_gap_days\n",
    "    ).reset_index(name=f'{prefix}_max_gap_days')\n",
    "    features_list.append(gap_days)\n",
    "    \n",
    "    # 交易间隔统计\n",
    "    def calc_trn_intervals(group):\n",
    "        dates = sorted(group['DATE'].unique())\n",
    "        if len(dates) <= 1:\n",
    "            return pd.Series({\n",
    "                'mean_interval': 0,\n",
    "                'std_interval': 0,\n",
    "                'min_interval': 0,\n",
    "                'max_interval': 0,\n",
    "                'median_interval': 0\n",
    "            })\n",
    "        intervals = [(dates[i] - dates[i-1]).days for i in range(1, len(dates))]\n",
    "        return pd.Series({\n",
    "            'mean_interval': np.mean(intervals),\n",
    "            'std_interval': np.std(intervals),\n",
    "            'min_interval': np.min(intervals),\n",
    "            'max_interval': np.max(intervals),\n",
    "            'median_interval': np.median(intervals)\n",
    "        })\n",
    "    \n",
    "    interval_stats = df.groupby('CUST_NO').apply(calc_trn_intervals).reset_index()\n",
    "    interval_stats.columns = ['CUST_NO',\n",
    "                             f'{prefix}_interval_mean',\n",
    "                             f'{prefix}_interval_std',\n",
    "                             f'{prefix}_interval_min',\n",
    "                             f'{prefix}_interval_max',\n",
    "                             f'{prefix}_interval_median']\n",
    "    features_list.append(interval_stats)\n",
    "    \n",
    "    # 间隔规律性(CV: Coefficient of Variation)\n",
    "    interval_stats[f'{prefix}_interval_cv'] = interval_stats[f'{prefix}_interval_std'] / (interval_stats[f'{prefix}_interval_mean'] + 1e-5)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 11. 高级统计特征\n",
    "    # ====================================================================================\n",
    "    print(\"11. 高级统计特征...\")\n",
    "    \n",
    "    from scipy.stats import skew, kurtosis\n",
    "    \n",
    "    # 金额偏度和峰度\n",
    "    skew_kurt = df.groupby('CUST_NO')['TFT_TRNAMT'].agg([\n",
    "        (f'{prefix}_amt_skew', lambda x: skew(x) if len(x) > 2 else 0),\n",
    "        (f'{prefix}_amt_kurt', lambda x: kurtosis(x) if len(x) > 2 else 0)\n",
    "    ]).reset_index()\n",
    "    features_list.append(skew_kurt)\n",
    "    \n",
    "    # 对数金额的偏度和峰度\n",
    "    skew_kurt_log = df.groupby('CUST_NO')['TFT_TRNAMT_log'].agg([\n",
    "        (f'{prefix}_amt_log_skew', lambda x: skew(x) if len(x) > 2 else 0),\n",
    "        (f'{prefix}_amt_log_kurt', lambda x: kurtosis(x) if len(x) > 2 else 0)\n",
    "    ]).reset_index()\n",
    "    features_list.append(skew_kurt_log)\n",
    "    \n",
    "    # 零金额交易\n",
    "    df_zero = df[df['TFT_TRNAMT'] == 0]\n",
    "    if len(df_zero) > 0:\n",
    "        zero_amt = df_zero.groupby('CUST_NO').size().reset_index(name=f'{prefix}_zero_amt_count')\n",
    "        features_list.append(zero_amt)\n",
    "    \n",
    "    # 相同金额交易(可能是固定支付)\n",
    "    def calc_duplicate_amounts(group):\n",
    "        amt_counts = group['TFT_TRNAMT'].value_counts()\n",
    "        if len(amt_counts) == 0:\n",
    "            return pd.Series({\n",
    "                'dup_amt_count': 0,\n",
    "                'dup_amt_max': 0,\n",
    "                'dup_amt_ratio': 0\n",
    "            })\n",
    "        max_dup = amt_counts.max()\n",
    "        dup_count = (amt_counts > 1).sum()\n",
    "        return pd.Series({\n",
    "            'dup_amt_count': dup_count,\n",
    "            'dup_amt_max': max_dup,\n",
    "            'dup_amt_ratio': max_dup / len(group) if len(group) > 0 else 0\n",
    "        })\n",
    "    \n",
    "    dup_amt_stats = df.groupby('CUST_NO').apply(calc_duplicate_amounts).reset_index()\n",
    "    dup_amt_stats.columns = ['CUST_NO', f'{prefix}_dup_amt_count', \n",
    "                            f'{prefix}_dup_amt_max', f'{prefix}_dup_amt_ratio']\n",
    "    features_list.append(dup_amt_stats)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 12. 交叉特征\n",
    "    # ====================================================================================\n",
    "    print(\"12. 交叉特征...\")\n",
    "    \n",
    "    # 业务代码 x 状态\n",
    "    cross_bsn_stt = df.groupby(['CUST_NO', 'TFT_STDBSNCOD', 'TFT_STT']).size().reset_index(name='count')\n",
    "    cross_bsn_stt_agg = cross_bsn_stt.groupby('CUST_NO')['count'].agg([\n",
    "        (f'{prefix}_bsn_stt_combinations', 'count'),\n",
    "        (f'{prefix}_bsn_stt_max_count', 'max'),\n",
    "        (f'{prefix}_bsn_stt_mean_count', 'mean')\n",
    "    ]).reset_index()\n",
    "    features_list.append(cross_bsn_stt_agg)\n",
    "    \n",
    "    # 业务代码 x 代收付类型\n",
    "    cross_bsn_pay = df.groupby(['CUST_NO', 'TFT_STDBSNCOD', 'TFT_PAYCOD']).size().reset_index(name='count')\n",
    "    cross_bsn_pay_agg = cross_bsn_pay.groupby('CUST_NO')['count'].agg([\n",
    "        (f'{prefix}_bsn_pay_combinations', 'count'),\n",
    "        (f'{prefix}_bsn_pay_max_count', 'max')\n",
    "    ]).reset_index()\n",
    "    features_list.append(cross_bsn_pay_agg)\n",
    "    \n",
    "    # 业务代码 x 客户类别\n",
    "    cross_bsn_cst = df.groupby(['CUST_NO', 'TFT_STDBSNCOD', 'TFT_CSTTYPE']).size().reset_index(name='count')\n",
    "    cross_bsn_cst_agg = cross_bsn_cst.groupby('CUST_NO')['count'].agg([\n",
    "        (f'{prefix}_bsn_cst_combinations', 'count'),\n",
    "        (f'{prefix}_bsn_cst_max_count', 'max')\n",
    "    ]).reset_index()\n",
    "    features_list.append(cross_bsn_cst_agg)\n",
    "    \n",
    "    # 状态 x 代收付类型\n",
    "    cross_stt_pay = df.groupby(['CUST_NO', 'TFT_STT', 'TFT_PAYCOD']).size().reset_index(name='count')\n",
    "    cross_stt_pay_agg = cross_stt_pay.groupby('CUST_NO')['count'].agg([\n",
    "        (f'{prefix}_stt_pay_combinations', 'count')\n",
    "    ]).reset_index()\n",
    "    features_list.append(cross_stt_pay_agg)\n",
    "    \n",
    "    # 金额档位 x 业务代码\n",
    "    cross_amt_bsn = df.groupby(['CUST_NO', 'amount_bin', 'TFT_STDBSNCOD']).size().reset_index(name='count')\n",
    "    cross_amt_bsn_agg = cross_amt_bsn.groupby('CUST_NO')['count'].agg([\n",
    "        (f'{prefix}_amt_bsn_combinations', 'count'),\n",
    "        (f'{prefix}_amt_bsn_max_count', 'max')\n",
    "    ]).reset_index()\n",
    "    features_list.append(cross_amt_bsn_agg)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 13. RFM特征\n",
    "    # ====================================================================================\n",
    "    print(\"13. RFM特征...\")\n",
    "    \n",
    "    rfm_features = df.groupby('CUST_NO').agg({\n",
    "        'DATE': lambda x: (reference_date - x.max()).days,  # Recency\n",
    "        'TFT_TRNAMT': ['count', 'sum']  # Frequency & Monetary\n",
    "    }).reset_index()\n",
    "    rfm_features.columns = ['CUST_NO', f'{prefix}_recency', f'{prefix}_frequency', f'{prefix}_monetary']\n",
    "    \n",
    "    # RFM评分(分成5档) - 使用更稳健的方法处理重复值\n",
    "    try:\n",
    "        rfm_features[f'{prefix}_recency_score'] = pd.qcut(rfm_features[f'{prefix}_recency'], 5, labels=False, duplicates='drop') + 1\n",
    "        # Recency越小越好，所以反转分数\n",
    "        rfm_features[f'{prefix}_recency_score'] = rfm_features[f'{prefix}_recency_score'].max() - rfm_features[f'{prefix}_recency_score'] + 1\n",
    "    except:\n",
    "        # 如果qcut失败，使用排序rank\n",
    "        rfm_features[f'{prefix}_recency_score'] = rfm_features[f'{prefix}_recency'].rank(ascending=False, method='dense')\n",
    "        rfm_features[f'{prefix}_recency_score'] = (rfm_features[f'{prefix}_recency_score'] / rfm_features[f'{prefix}_recency_score'].max() * 5).fillna(3)\n",
    "    \n",
    "    try:\n",
    "        rfm_features[f'{prefix}_frequency_score'] = pd.qcut(rfm_features[f'{prefix}_frequency'], 5, labels=False, duplicates='drop') + 1\n",
    "    except:\n",
    "        rfm_features[f'{prefix}_frequency_score'] = rfm_features[f'{prefix}_frequency'].rank(ascending=True, method='dense')\n",
    "        rfm_features[f'{prefix}_frequency_score'] = (rfm_features[f'{prefix}_frequency_score'] / rfm_features[f'{prefix}_frequency_score'].max() * 5).fillna(3)\n",
    "    \n",
    "    try:\n",
    "        rfm_features[f'{prefix}_monetary_score'] = pd.qcut(rfm_features[f'{prefix}_monetary'], 5, labels=False, duplicates='drop') + 1\n",
    "    except:\n",
    "        rfm_features[f'{prefix}_monetary_score'] = rfm_features[f'{prefix}_monetary'].rank(ascending=True, method='dense')\n",
    "        rfm_features[f'{prefix}_monetary_score'] = (rfm_features[f'{prefix}_monetary_score'] / rfm_features[f'{prefix}_monetary_score'].max() * 5).fillna(3)\n",
    "    \n",
    "    # RFM综合得分\n",
    "    rfm_features[f'{prefix}_rfm_score'] = (rfm_features[f'{prefix}_recency_score'] + \n",
    "                                           rfm_features[f'{prefix}_frequency_score'] + \n",
    "                                           rfm_features[f'{prefix}_monetary_score'])\n",
    "    \n",
    "    features_list.append(rfm_features)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 14. 异常特征\n",
    "    # ====================================================================================\n",
    "    print(\"14. 异常特征...\")\n",
    "    \n",
    "    # 异常大额交易(超过均值+3倍标准差)\n",
    "    def calc_outlier_features(group):\n",
    "        mean_amt = group['TFT_TRNAMT'].mean()\n",
    "        std_amt = group['TFT_TRNAMT'].std()\n",
    "        threshold_high = mean_amt + 3 * std_amt\n",
    "        threshold_low = mean_amt - 3 * std_amt if mean_amt - 3 * std_amt > 0 else 0\n",
    "        \n",
    "        outlier_high_count = (group['TFT_TRNAMT'] > threshold_high).sum()\n",
    "        outlier_low_count = (group['TFT_TRNAMT'] < threshold_low).sum()\n",
    "        \n",
    "        return pd.Series({\n",
    "            'outlier_high_count': outlier_high_count,\n",
    "            'outlier_low_count': outlier_low_count,\n",
    "            'outlier_total_count': outlier_high_count + outlier_low_count,\n",
    "            'outlier_ratio': (outlier_high_count + outlier_low_count) / len(group) if len(group) > 0 else 0\n",
    "        })\n",
    "    \n",
    "    outlier_stats = df.groupby('CUST_NO').apply(calc_outlier_features).reset_index()\n",
    "    outlier_stats.columns = ['CUST_NO', f'{prefix}_outlier_high_count', \n",
    "                            f'{prefix}_outlier_low_count', f'{prefix}_outlier_total_count',\n",
    "                            f'{prefix}_outlier_ratio']\n",
    "    features_list.append(outlier_stats)\n",
    "    \n",
    "    # 异常频率(某天交易次数过多)\n",
    "    daily_count = df.groupby(['CUST_NO', 'DATE']).size().reset_index(name='daily_count')\n",
    "    daily_outlier = daily_count.groupby('CUST_NO')['daily_count'].agg([\n",
    "        (f'{prefix}_daily_count_max', 'max'),\n",
    "        (f'{prefix}_daily_count_mean', 'mean'),\n",
    "        (f'{prefix}_daily_count_std', 'std')\n",
    "    ]).reset_index()\n",
    "    daily_outlier[f'{prefix}_daily_freq_outlier'] = (daily_outlier[f'{prefix}_daily_count_max'] > \n",
    "                                                      daily_outlier[f'{prefix}_daily_count_mean'] + \n",
    "                                                      3 * daily_outlier[f'{prefix}_daily_count_std']).astype(int)\n",
    "    features_list.append(daily_outlier)\n",
    "    \n",
    "    # ====================================================================================\n",
    "    # 15. 时序变化特征\n",
    "    # ====================================================================================\n",
    "    print(\"15. 时序变化特征...\")\n",
    "    \n",
    "    # 按时间序列计算移动平均和变化率\n",
    "    df_sorted = df.sort_values(['CUST_NO', 'DATE'])\n",
    "    \n",
    "    # 近期交易金额趋势(最近10笔vs之前10笔)\n",
    "    def calc_recent_trend(group):\n",
    "        if len(group) < 10:\n",
    "            return pd.Series({\n",
    "                'recent_amt_change': 0,\n",
    "                'recent_amt_mean': 0,\n",
    "                'previous_amt_mean': 0\n",
    "            })\n",
    "        \n",
    "        recent_10 = group.tail(10)\n",
    "        previous_10 = group.iloc[-20:-10] if len(group) >= 20 else group.head(10)\n",
    "        \n",
    "        recent_amt_mean = recent_10['TFT_TRNAMT'].mean()\n",
    "        previous_amt_mean = previous_10['TFT_TRNAMT'].mean()\n",
    "        \n",
    "        amt_change = (recent_amt_mean - previous_amt_mean) / (previous_amt_mean + 1e-5)\n",
    "        \n",
    "        return pd.Series({\n",
    "            'recent_amt_change': amt_change,\n",
    "            'recent_amt_mean': recent_amt_mean,\n",
    "            'previous_amt_mean': previous_amt_mean\n",
    "        })\n",
    "    \n",
    "    recent_trend = df_sorted.groupby('CUST_NO').apply(calc_recent_trend).reset_index()\n",
    "    recent_trend.columns = ['CUST_NO', f'{prefix}_recent_amt_change', \n",
    "                           f'{prefix}_recent_amt_mean', f'{prefix}_previous_amt_mean']\n",
    "    features_list.append(recent_trend)\n",
    "    \n",
    "    # 交易金额波动率(类似金融中的波动率)\n",
    "    def calc_volatility(group):\n",
    "        if len(group) < 2:\n",
    "            return 0\n",
    "        returns = group['TFT_TRNAMT'].pct_change().dropna()\n",
    "        if len(returns) == 0:\n",
    "            return 0\n",
    "        return returns.std()\n",
    "    \n",
    "    volatility = df_sorted.groupby('CUST_NO').apply(calc_volatility).reset_index(name=f'{prefix}_amt_volatility')\n",
    "    features_list.append(volatility)\n",
    "    \n",
    "    # 首末交易金额对比\n",
    "    first_last_amt = df_sorted.groupby('CUST_NO').agg({\n",
    "        'TFT_TRNAMT': ['first', 'last']\n",
    "    }).reset_index()\n",
    "    first_last_amt.columns = ['CUST_NO', f'{prefix}_first_amt', f'{prefix}_last_amt']\n",
    "    first_last_amt[f'{prefix}_first_last_amt_ratio'] = first_last_amt[f'{prefix}_last_amt'] / (first_last_amt[f'{prefix}_first_amt'] + 1e-5)\n",
    "    first_last_amt[f'{prefix}_first_last_amt_diff'] = first_last_amt[f'{prefix}_last_amt'] - first_last_amt[f'{prefix}_first_amt']\n",
    "    features_list.append(first_last_amt)\n",
    "    \n",
    "    print(f\"第二部分特征生成完成，当前共有 {len(features_list)} 组特征\")\n",
    "    \n",
    "    return features_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c393d7fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_all_features(features_list, prefix='mb_trnflw'):\n",
    "    \"\"\"\n",
    "    合并所有特征\n",
    "    \n",
    "    参数:\n",
    "    - features_list: 特征DataFrame列表\n",
    "    - prefix: 特征前缀\n",
    "    \n",
    "    返回:\n",
    "    - 合并后的特征DataFrame\n",
    "    \"\"\"\n",
    "    print(\"=\" * 80)\n",
    "    print(\"开始合并所有特征...\")\n",
    "    \n",
    "    if len(features_list) == 0:\n",
    "        print(\"警告: 没有特征可以合并!\")\n",
    "        return pd.DataFrame()\n",
    "    \n",
    "    features = features_list[0]\n",
    "    for i, feat_df in enumerate(features_list[1:], 1):\n",
    "        if len(feat_df) > 0:\n",
    "            features = features.merge(feat_df, on='CUST_NO', how='left')\n",
    "            if i % 10 == 0:\n",
    "                print(f\"已合并 {i}/{len(features_list)-1} 组特征, 当前特征数: {len(features.columns)-1}\")\n",
    "    \n",
    "    # 填充缺失值\n",
    "    print(\"填充缺失值...\")\n",
    "    features = features.fillna(0)\n",
    "    \n",
    "    # 添加衍生比率特征\n",
    "    print(\"生成衍生比率特征...\")\n",
    "    \n",
    "    # 周末比率\n",
    "    if f'{prefix}_weekend_count' in features.columns and f'{prefix}_count' in features.columns:\n",
    "        features[f'{prefix}_weekend_ratio'] = features[f'{prefix}_weekend_count'] / (features[f'{prefix}_count'] + 1e-5)\n",
    "    \n",
    "    if f'{prefix}_weekend_amt_sum' in features.columns and f'{prefix}_amt_sum' in features.columns:\n",
    "        features[f'{prefix}_weekend_amt_ratio'] = features[f'{prefix}_weekend_amt_sum'] / (features[f'{prefix}_amt_sum'] + 1e-5)\n",
    "    \n",
    "    # 大额交易比率\n",
    "    if f'{prefix}_large_amt_count' in features.columns and f'{prefix}_count' in features.columns:\n",
    "        features[f'{prefix}_large_amt_count_ratio'] = features[f'{prefix}_large_amt_count'] / (features[f'{prefix}_count'] + 1e-5)\n",
    "    \n",
    "    if f'{prefix}_large_amt_sum' in features.columns and f'{prefix}_amt_sum' in features.columns:\n",
    "        features[f'{prefix}_large_amt_sum_ratio'] = features[f'{prefix}_large_amt_sum'] / (features[f'{prefix}_amt_sum'] + 1e-5)\n",
    "    \n",
    "    # 小额交易比率\n",
    "    if f'{prefix}_small_amt_count' in features.columns and f'{prefix}_count' in features.columns:\n",
    "        features[f'{prefix}_small_amt_count_ratio'] = features[f'{prefix}_small_amt_count'] / (features[f'{prefix}_count'] + 1e-5)\n",
    "    \n",
    "    # 成功率\n",
    "    if f'{prefix}_success_count' in features.columns and f'{prefix}_count' in features.columns:\n",
    "        features[f'{prefix}_success_ratio'] = features[f'{prefix}_success_count'] / (features[f'{prefix}_count'] + 1e-5)\n",
    "    \n",
    "    # 零金额比率\n",
    "    if f'{prefix}_zero_amt_count' in features.columns and f'{prefix}_count' in features.columns:\n",
    "        features[f'{prefix}_zero_amt_ratio'] = features[f'{prefix}_zero_amt_count'] / (features[f'{prefix}_count'] + 1e-5)\n",
    "    \n",
    "    # 月初月末比率\n",
    "    if f'{prefix}_month_start_count' in features.columns and f'{prefix}_count' in features.columns:\n",
    "        features[f'{prefix}_month_start_ratio'] = features[f'{prefix}_month_start_count'] / (features[f'{prefix}_count'] + 1e-5)\n",
    "    \n",
    "    if f'{prefix}_month_end_count' in features.columns and f'{prefix}_count' in features.columns:\n",
    "        features[f'{prefix}_month_end_ratio'] = features[f'{prefix}_month_end_count'] / (features[f'{prefix}_count'] + 1e-5)\n",
    "    \n",
    "    if f'{prefix}_mid_month_count' in features.columns and f'{prefix}_count' in features.columns:\n",
    "        features[f'{prefix}_mid_month_ratio'] = features[f'{prefix}_mid_month_count'] / (features[f'{prefix}_count'] + 1e-5)\n",
    "    \n",
    "    # 活跃度相关比率\n",
    "    if f'{prefix}_max_consecutive_days' in features.columns and f'{prefix}_active_span' in features.columns:\n",
    "        features[f'{prefix}_consecutive_ratio'] = features[f'{prefix}_max_consecutive_days'] / (features[f'{prefix}_active_span'] + 1e-5)\n",
    "    \n",
    "    if f'{prefix}_max_gap_days' in features.columns and f'{prefix}_active_span' in features.columns:\n",
    "        features[f'{prefix}_gap_ratio'] = features[f'{prefix}_max_gap_days'] / (features[f'{prefix}_active_span'] + 1e-5)\n",
    "    \n",
    "    # 业务代码集中度vs多样性\n",
    "    if f'{prefix}_bsncod_concentration' in features.columns and f'{prefix}_bsncod_entropy' in features.columns:\n",
    "        features[f'{prefix}_bsncod_concentration_entropy_ratio'] = features[f'{prefix}_bsncod_concentration'] / (features[f'{prefix}_bsncod_entropy'] + 1e-5)\n",
    "    \n",
    "    # 确保没有无穷值和NaN\n",
    "    print(\"处理无穷值...\")\n",
    "    features = features.replace([np.inf, -np.inf], 0)\n",
    "    features = features.fillna(0)\n",
    "    \n",
    "    print(\"=\" * 80)\n",
    "    print(f\"特征合并完成!\")\n",
    "    print(f\"客户数: {len(features)}\")\n",
    "    print(f\"特征数: {len(features.columns) - 1}\")\n",
    "    print(\"=\" * 80)\n",
    "    \n",
    "    return features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd27c3f5",
   "metadata": {},
   "source": [
    "### 生成训练集特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d1fb6097",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始生成训练集特征...\n",
      "================================================================================\n",
      "开始构建mb_trnflw特征...\n",
      "================================================================================\n",
      "1. 基础统计特征...\n",
      "2. 时间窗口特征...\n",
      "3. 周末vs工作日特征...\n",
      "4. 标准业务代码特征...\n",
      "2. 时间窗口特征...\n",
      "3. 周末vs工作日特征...\n",
      "4. 标准业务代码特征...\n",
      "5. 状态特征...\n",
      "6. 代收付类型特征...\n",
      "5. 状态特征...\n",
      "6. 代收付类型特征...\n",
      "7. 客户类别特征...\n",
      "8. 金额分档特征...\n",
      "当前已生成 51 组特征\n",
      "9. 时间周期特征...\n",
      "10. 连续性与规律性特征...\n",
      "7. 客户类别特征...\n",
      "8. 金额分档特征...\n",
      "当前已生成 51 组特征\n",
      "9. 时间周期特征...\n",
      "10. 连续性与规律性特征...\n",
      "11. 高级统计特征...\n",
      "11. 高级统计特征...\n",
      "12. 交叉特征...\n",
      "13. RFM特征...\n",
      "14. 异常特征...\n",
      "12. 交叉特征...\n",
      "13. RFM特征...\n",
      "14. 异常特征...\n",
      "15. 时序变化特征...\n",
      "第二部分特征生成完成，当前共有 75 组特征\n",
      "================================================================================\n",
      "开始合并所有特征...\n",
      "已合并 10/74 组特征, 当前特征数: 157\n",
      "已合并 20/74 组特征, 当前特征数: 197\n",
      "15. 时序变化特征...\n",
      "第二部分特征生成完成，当前共有 75 组特征\n",
      "================================================================================\n",
      "开始合并所有特征...\n",
      "已合并 10/74 组特征, 当前特征数: 157\n",
      "已合并 20/74 组特征, 当前特征数: 197\n",
      "已合并 30/74 组特征, 当前特征数: 222\n",
      "已合并 40/74 组特征, 当前特征数: 254\n",
      "已合并 50/74 组特征, 当前特征数: 275\n",
      "已合并 60/74 组特征, 当前特征数: 310\n",
      "已合并 70/74 组特征, 当前特征数: 337\n",
      "填充缺失值...\n",
      "生成衍生比率特征...\n",
      "处理无穷值...\n",
      "================================================================================\n",
      "特征合并完成!\n",
      "客户数: 1291\n",
      "特征数: 358\n",
      "================================================================================\n",
      "\n",
      "训练集特征生成完成!\n",
      "特征维度: (1291, 359)\n",
      "前5行预览:\n",
      "                            CUST_NO  mb_trnflw_count  mb_trnflw_amt_sum  \\\n",
      "0  00249bd7ed04c80b2d3dc8c5f4618392                1            10000.0   \n",
      "1  007984a77d0949609acf63b7dd9029c6                1             4000.0   \n",
      "2  009a28da619977ebf8210dffd8674f58                1             5000.0   \n",
      "3  00eef8ac82ef7b188a40bc1cebe7b7e6                1              300.0   \n",
      "4  00f9f751e2b60ac809487923b79dbee6                1             3663.0   \n",
      "\n",
      "   mb_trnflw_amt_mean  mb_trnflw_amt_median  mb_trnflw_amt_std  \\\n",
      "0             10000.0               10000.0                0.0   \n",
      "1              4000.0                4000.0                0.0   \n",
      "2              5000.0                5000.0                0.0   \n",
      "3               300.0                 300.0                0.0   \n",
      "4              3663.0                3663.0                0.0   \n",
      "\n",
      "   mb_trnflw_amt_min  mb_trnflw_amt_max  mb_trnflw_amt_q25  mb_trnflw_amt_q75  \\\n",
      "0            10000.0            10000.0            10000.0            10000.0   \n",
      "1             4000.0             4000.0             4000.0             4000.0   \n",
      "2             5000.0             5000.0             5000.0             5000.0   \n",
      "3              300.0              300.0              300.0              300.0   \n",
      "4             3663.0             3663.0             3663.0             3663.0   \n",
      "\n",
      "   ...  mb_trnflw_first_last_amt_diff  mb_trnflw_large_amt_count_ratio  \\\n",
      "0  ...                            0.0                              0.0   \n",
      "1  ...                            0.0                              0.0   \n",
      "2  ...                            0.0                              0.0   \n",
      "3  ...                            0.0                              0.0   \n",
      "4  ...                            0.0                              0.0   \n",
      "\n",
      "   mb_trnflw_large_amt_sum_ratio  mb_trnflw_small_amt_count_ratio  \\\n",
      "0                            0.0                              0.0   \n",
      "1                            0.0                              0.0   \n",
      "2                            0.0                              0.0   \n",
      "3                            0.0                              0.0   \n",
      "4                            0.0                              0.0   \n",
      "\n",
      "   mb_trnflw_success_ratio  mb_trnflw_zero_amt_ratio  \\\n",
      "0                  0.99999                       0.0   \n",
      "1                  0.99999                       0.0   \n",
      "2                  0.99999                       0.0   \n",
      "3                  0.99999                       0.0   \n",
      "4                  0.99999                       0.0   \n",
      "\n",
      "   mb_trnflw_mid_month_ratio  mb_trnflw_consecutive_ratio  \\\n",
      "0                    0.99999                      0.99999   \n",
      "1                    0.99999                      0.99999   \n",
      "2                    0.99999                      0.99999   \n",
      "3                    0.99999                      0.99999   \n",
      "4                    0.99999                      0.99999   \n",
      "\n",
      "   mb_trnflw_gap_ratio  mb_trnflw_bsncod_concentration_entropy_ratio  \n",
      "0                  0.0                                  100001.00001  \n",
      "1                  0.0                                  100001.00001  \n",
      "2                  0.0                                  100001.00001  \n",
      "3                  0.0                                  100001.00001  \n",
      "4                  0.0                                  100001.00001  \n",
      "\n",
      "[5 rows x 359 columns]\n",
      "已合并 30/74 组特征, 当前特征数: 222\n",
      "已合并 40/74 组特征, 当前特征数: 254\n",
      "已合并 50/74 组特征, 当前特征数: 275\n",
      "已合并 60/74 组特征, 当前特征数: 310\n",
      "已合并 70/74 组特征, 当前特征数: 337\n",
      "填充缺失值...\n",
      "生成衍生比率特征...\n",
      "处理无穷值...\n",
      "================================================================================\n",
      "特征合并完成!\n",
      "客户数: 1291\n",
      "特征数: 358\n",
      "================================================================================\n",
      "\n",
      "训练集特征生成完成!\n",
      "特征维度: (1291, 359)\n",
      "前5行预览:\n",
      "                            CUST_NO  mb_trnflw_count  mb_trnflw_amt_sum  \\\n",
      "0  00249bd7ed04c80b2d3dc8c5f4618392                1            10000.0   \n",
      "1  007984a77d0949609acf63b7dd9029c6                1             4000.0   \n",
      "2  009a28da619977ebf8210dffd8674f58                1             5000.0   \n",
      "3  00eef8ac82ef7b188a40bc1cebe7b7e6                1              300.0   \n",
      "4  00f9f751e2b60ac809487923b79dbee6                1             3663.0   \n",
      "\n",
      "   mb_trnflw_amt_mean  mb_trnflw_amt_median  mb_trnflw_amt_std  \\\n",
      "0             10000.0               10000.0                0.0   \n",
      "1              4000.0                4000.0                0.0   \n",
      "2              5000.0                5000.0                0.0   \n",
      "3               300.0                 300.0                0.0   \n",
      "4              3663.0                3663.0                0.0   \n",
      "\n",
      "   mb_trnflw_amt_min  mb_trnflw_amt_max  mb_trnflw_amt_q25  mb_trnflw_amt_q75  \\\n",
      "0            10000.0            10000.0            10000.0            10000.0   \n",
      "1             4000.0             4000.0             4000.0             4000.0   \n",
      "2             5000.0             5000.0             5000.0             5000.0   \n",
      "3              300.0              300.0              300.0              300.0   \n",
      "4             3663.0             3663.0             3663.0             3663.0   \n",
      "\n",
      "   ...  mb_trnflw_first_last_amt_diff  mb_trnflw_large_amt_count_ratio  \\\n",
      "0  ...                            0.0                              0.0   \n",
      "1  ...                            0.0                              0.0   \n",
      "2  ...                            0.0                              0.0   \n",
      "3  ...                            0.0                              0.0   \n",
      "4  ...                            0.0                              0.0   \n",
      "\n",
      "   mb_trnflw_large_amt_sum_ratio  mb_trnflw_small_amt_count_ratio  \\\n",
      "0                            0.0                              0.0   \n",
      "1                            0.0                              0.0   \n",
      "2                            0.0                              0.0   \n",
      "3                            0.0                              0.0   \n",
      "4                            0.0                              0.0   \n",
      "\n",
      "   mb_trnflw_success_ratio  mb_trnflw_zero_amt_ratio  \\\n",
      "0                  0.99999                       0.0   \n",
      "1                  0.99999                       0.0   \n",
      "2                  0.99999                       0.0   \n",
      "3                  0.99999                       0.0   \n",
      "4                  0.99999                       0.0   \n",
      "\n",
      "   mb_trnflw_mid_month_ratio  mb_trnflw_consecutive_ratio  \\\n",
      "0                    0.99999                      0.99999   \n",
      "1                    0.99999                      0.99999   \n",
      "2                    0.99999                      0.99999   \n",
      "3                    0.99999                      0.99999   \n",
      "4                    0.99999                      0.99999   \n",
      "\n",
      "   mb_trnflw_gap_ratio  mb_trnflw_bsncod_concentration_entropy_ratio  \n",
      "0                  0.0                                  100001.00001  \n",
      "1                  0.0                                  100001.00001  \n",
      "2                  0.0                                  100001.00001  \n",
      "3                  0.0                                  100001.00001  \n",
      "4                  0.0                                  100001.00001  \n",
      "\n",
      "[5 rows x 359 columns]\n"
     ]
    }
   ],
   "source": [
    "print(\"开始生成训练集特征...\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "# 第一部分特征\n",
    "train_features_list = create_mb_trnflw_features(train_df, train_ref_date, prefix='mb_trnflw')\n",
    "\n",
    "# 第二部分特征\n",
    "train_features_list = create_mb_trnflw_features_part2(train_df, train_ref_date, train_features_list, prefix='mb_trnflw')\n",
    "\n",
    "# 合并所有特征\n",
    "train_mb_trnflw_features = merge_all_features(train_features_list, prefix='mb_trnflw')\n",
    "\n",
    "print(\"\\n训练集特征生成完成!\")\n",
    "print(f\"特征维度: {train_mb_trnflw_features.shape}\")\n",
    "print(f\"前5行预览:\")\n",
    "print(train_mb_trnflw_features.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "214b2db2",
   "metadata": {},
   "source": [
    "### 生成测试集特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "fc5517a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始生成测试集特征...\n",
      "================================================================================\n",
      "开始构建mb_trnflw特征...\n",
      "================================================================================\n",
      "1. 基础统计特征...\n",
      "2. 时间窗口特征...\n",
      "3. 周末vs工作日特征...\n",
      "4. 标准业务代码特征...\n",
      "5. 状态特征...\n",
      "6. 代收付类型特征...\n",
      "7. 客户类别特征...\n",
      "2. 时间窗口特征...\n",
      "3. 周末vs工作日特征...\n",
      "4. 标准业务代码特征...\n",
      "5. 状态特征...\n",
      "6. 代收付类型特征...\n",
      "7. 客户类别特征...\n",
      "8. 金额分档特征...\n",
      "当前已生成 49 组特征\n",
      "9. 时间周期特征...\n",
      "10. 连续性与规律性特征...\n",
      "11. 高级统计特征...\n",
      "8. 金额分档特征...\n",
      "当前已生成 49 组特征\n",
      "9. 时间周期特征...\n",
      "10. 连续性与规律性特征...\n",
      "11. 高级统计特征...\n",
      "12. 交叉特征...\n",
      "13. RFM特征...\n",
      "14. 异常特征...\n",
      "15. 时序变化特征...\n",
      "第二部分特征生成完成，当前共有 76 组特征\n",
      "================================================================================\n",
      "开始合并所有特征...\n",
      "12. 交叉特征...\n",
      "13. RFM特征...\n",
      "14. 异常特征...\n",
      "15. 时序变化特征...\n",
      "第二部分特征生成完成，当前共有 76 组特征\n",
      "================================================================================\n",
      "开始合并所有特征...\n",
      "已合并 10/75 组特征, 当前特征数: 157\n",
      "已合并 20/75 组特征, 当前特征数: 197\n",
      "已合并 30/75 组特征, 当前特征数: 222\n",
      "已合并 40/75 组特征, 当前特征数: 250\n",
      "已合并 50/75 组特征, 当前特征数: 277\n",
      "已合并 60/75 组特征, 当前特征数: 309\n",
      "已合并 70/75 组特征, 当前特征数: 334\n",
      "填充缺失值...\n",
      "生成衍生比率特征...\n",
      "处理无穷值...\n",
      "================================================================================\n",
      "特征合并完成!\n",
      "客户数: 457\n",
      "特征数: 360\n",
      "================================================================================\n",
      "\n",
      "测试集特征生成完成!\n",
      "特征维度: (457, 361)\n",
      "前5行预览:\n",
      "                            CUST_NO  mb_trnflw_count  mb_trnflw_amt_sum  \\\n",
      "0  006e7d6eb9553ddfd568217f32eec80f                1             3000.0   \n",
      "1  00ccb53485d08334e01e322a8aca1ec9                3           117000.0   \n",
      "2  01459bab682a54e2b752cee864f2e676                1              500.0   \n",
      "3  01ee82d78213082847766d7058ad4946                3            50120.0   \n",
      "4  020765442663627f49f6111da3f55716                1             9278.0   \n",
      "\n",
      "   mb_trnflw_amt_mean  mb_trnflw_amt_median  mb_trnflw_amt_std  \\\n",
      "0         3000.000000                3000.0           0.000000   \n",
      "1        39000.000000               50000.0       19052.558883   \n",
      "2          500.000000                 500.0           0.000000   \n",
      "3        16706.666667               17500.0       10532.432451   \n",
      "4         9278.000000                9278.0           0.000000   \n",
      "\n",
      "   mb_trnflw_amt_min  mb_trnflw_amt_max  mb_trnflw_amt_q25  mb_trnflw_amt_q75  \\\n",
      "0             3000.0             3000.0             3000.0             3000.0   \n",
      "1            17000.0            50000.0            33500.0            50000.0   \n",
      "2              500.0              500.0              500.0              500.0   \n",
      "3             5800.0            26820.0            11650.0            22160.0   \n",
      "4             9278.0             9278.0             9278.0             9278.0   \n",
      "\n",
      "   ...  mb_trnflw_large_amt_count_ratio  mb_trnflw_large_amt_sum_ratio  \\\n",
      "0  ...                              0.0                            0.0   \n",
      "1  ...                              0.0                            0.0   \n",
      "2  ...                              0.0                            0.0   \n",
      "3  ...                              0.0                            0.0   \n",
      "4  ...                              0.0                            0.0   \n",
      "\n",
      "   mb_trnflw_small_amt_count_ratio  mb_trnflw_success_ratio  \\\n",
      "0                              0.0                 0.999990   \n",
      "1                              0.0                 0.666664   \n",
      "2                              0.0                 0.999990   \n",
      "3                              0.0                 0.999997   \n",
      "4                              0.0                 0.999990   \n",
      "\n",
      "   mb_trnflw_zero_amt_ratio  mb_trnflw_month_start_ratio  \\\n",
      "0                       0.0                     0.000000   \n",
      "1                       0.0                     0.000000   \n",
      "2                       0.0                     0.000000   \n",
      "3                       0.0                     0.333332   \n",
      "4                       0.0                     0.000000   \n",
      "\n",
      "   mb_trnflw_mid_month_ratio  mb_trnflw_consecutive_ratio  \\\n",
      "0                   0.000000                     0.999990   \n",
      "1                   0.999997                     0.249999   \n",
      "2                   0.000000                     0.999990   \n",
      "3                   0.333332                     0.028571   \n",
      "4                   0.999990                     0.999990   \n",
      "\n",
      "   mb_trnflw_gap_ratio  mb_trnflw_bsncod_concentration_entropy_ratio  \n",
      "0             0.000000                                  100001.00001  \n",
      "1             0.499999                                  100001.00001  \n",
      "2             0.000000                                  100001.00001  \n",
      "3             0.485714                                  100001.00001  \n",
      "4             0.000000                                  100001.00001  \n",
      "\n",
      "[5 rows x 361 columns]\n",
      "已合并 10/75 组特征, 当前特征数: 157\n",
      "已合并 20/75 组特征, 当前特征数: 197\n",
      "已合并 30/75 组特征, 当前特征数: 222\n",
      "已合并 40/75 组特征, 当前特征数: 250\n",
      "已合并 50/75 组特征, 当前特征数: 277\n",
      "已合并 60/75 组特征, 当前特征数: 309\n",
      "已合并 70/75 组特征, 当前特征数: 334\n",
      "填充缺失值...\n",
      "生成衍生比率特征...\n",
      "处理无穷值...\n",
      "================================================================================\n",
      "特征合并完成!\n",
      "客户数: 457\n",
      "特征数: 360\n",
      "================================================================================\n",
      "\n",
      "测试集特征生成完成!\n",
      "特征维度: (457, 361)\n",
      "前5行预览:\n",
      "                            CUST_NO  mb_trnflw_count  mb_trnflw_amt_sum  \\\n",
      "0  006e7d6eb9553ddfd568217f32eec80f                1             3000.0   \n",
      "1  00ccb53485d08334e01e322a8aca1ec9                3           117000.0   \n",
      "2  01459bab682a54e2b752cee864f2e676                1              500.0   \n",
      "3  01ee82d78213082847766d7058ad4946                3            50120.0   \n",
      "4  020765442663627f49f6111da3f55716                1             9278.0   \n",
      "\n",
      "   mb_trnflw_amt_mean  mb_trnflw_amt_median  mb_trnflw_amt_std  \\\n",
      "0         3000.000000                3000.0           0.000000   \n",
      "1        39000.000000               50000.0       19052.558883   \n",
      "2          500.000000                 500.0           0.000000   \n",
      "3        16706.666667               17500.0       10532.432451   \n",
      "4         9278.000000                9278.0           0.000000   \n",
      "\n",
      "   mb_trnflw_amt_min  mb_trnflw_amt_max  mb_trnflw_amt_q25  mb_trnflw_amt_q75  \\\n",
      "0             3000.0             3000.0             3000.0             3000.0   \n",
      "1            17000.0            50000.0            33500.0            50000.0   \n",
      "2              500.0              500.0              500.0              500.0   \n",
      "3             5800.0            26820.0            11650.0            22160.0   \n",
      "4             9278.0             9278.0             9278.0             9278.0   \n",
      "\n",
      "   ...  mb_trnflw_large_amt_count_ratio  mb_trnflw_large_amt_sum_ratio  \\\n",
      "0  ...                              0.0                            0.0   \n",
      "1  ...                              0.0                            0.0   \n",
      "2  ...                              0.0                            0.0   \n",
      "3  ...                              0.0                            0.0   \n",
      "4  ...                              0.0                            0.0   \n",
      "\n",
      "   mb_trnflw_small_amt_count_ratio  mb_trnflw_success_ratio  \\\n",
      "0                              0.0                 0.999990   \n",
      "1                              0.0                 0.666664   \n",
      "2                              0.0                 0.999990   \n",
      "3                              0.0                 0.999997   \n",
      "4                              0.0                 0.999990   \n",
      "\n",
      "   mb_trnflw_zero_amt_ratio  mb_trnflw_month_start_ratio  \\\n",
      "0                       0.0                     0.000000   \n",
      "1                       0.0                     0.000000   \n",
      "2                       0.0                     0.000000   \n",
      "3                       0.0                     0.333332   \n",
      "4                       0.0                     0.000000   \n",
      "\n",
      "   mb_trnflw_mid_month_ratio  mb_trnflw_consecutive_ratio  \\\n",
      "0                   0.000000                     0.999990   \n",
      "1                   0.999997                     0.249999   \n",
      "2                   0.000000                     0.999990   \n",
      "3                   0.333332                     0.028571   \n",
      "4                   0.999990                     0.999990   \n",
      "\n",
      "   mb_trnflw_gap_ratio  mb_trnflw_bsncod_concentration_entropy_ratio  \n",
      "0             0.000000                                  100001.00001  \n",
      "1             0.499999                                  100001.00001  \n",
      "2             0.000000                                  100001.00001  \n",
      "3             0.485714                                  100001.00001  \n",
      "4             0.000000                                  100001.00001  \n",
      "\n",
      "[5 rows x 361 columns]\n"
     ]
    }
   ],
   "source": [
    "print(\"开始生成测试集特征...\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "# 第一部分特征\n",
    "test_features_list = create_mb_trnflw_features(test_df, test_ref_date, prefix='mb_trnflw')\n",
    "\n",
    "# 第二部分特征\n",
    "test_features_list = create_mb_trnflw_features_part2(test_df, test_ref_date, test_features_list, prefix='mb_trnflw')\n",
    "\n",
    "# 合并所有特征\n",
    "test_mb_trnflw_features = merge_all_features(test_features_list, prefix='mb_trnflw')\n",
    "\n",
    "print(\"\\n测试集特征生成完成!\")\n",
    "print(f\"特征维度: {test_mb_trnflw_features.shape}\")\n",
    "print(f\"前5行预览:\")\n",
    "print(test_mb_trnflw_features.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6988274",
   "metadata": {},
   "source": [
    "### 特征一致性检查"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d6bc065c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "特征一致性检查\n",
      "================================================================================\n",
      "\n",
      "训练集特征数: 358\n",
      "测试集特征数: 360\n",
      "\n",
      "仅在训练集中的特征(8个):\n",
      "  - mb_trnflw_paycod_7_amt_max\n",
      "  - mb_trnflw_paycod_7_amt_mean\n",
      "  - mb_trnflw_paycod_7_amt_sum\n",
      "  - mb_trnflw_paycod_7_count\n",
      "  - mb_trnflw_paycod_8_amt_max\n",
      "  - mb_trnflw_paycod_8_amt_mean\n",
      "  - mb_trnflw_paycod_8_amt_sum\n",
      "  - mb_trnflw_paycod_8_count\n",
      "\n",
      "仅在测试集中的特征(10个):\n",
      "  - mb_trnflw_month_start_amt_mean\n",
      "  - mb_trnflw_month_start_amt_sum\n",
      "  - mb_trnflw_month_start_count\n",
      "  - mb_trnflw_month_start_ratio\n",
      "  - mb_trnflw_weekday2_amt_mean\n",
      "  - mb_trnflw_weekday2_amt_sum\n",
      "  - mb_trnflw_weekday2_count\n",
      "  - mb_trnflw_weekday3_amt_mean\n",
      "  - mb_trnflw_weekday3_amt_sum\n",
      "  - mb_trnflw_weekday3_count\n",
      "\n",
      "正在对齐特征...\n",
      "特征对齐完成!\n",
      "对齐后训练集特征数: 368\n",
      "对齐后测试集特征数: 368\n",
      "\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "print(\"=\" * 80)\n",
    "print(\"特征一致性检查\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "train_cols = set(train_mb_trnflw_features.columns)\n",
    "test_cols = set(test_mb_trnflw_features.columns)\n",
    "\n",
    "print(f\"\\n训练集特征数: {len(train_cols) - 1}\")\n",
    "print(f\"测试集特征数: {len(test_cols) - 1}\")\n",
    "\n",
    "if train_cols == test_cols:\n",
    "    print(\"\\n训练集和测试集特征完全一致!\")\n",
    "else:\n",
    "    only_train = train_cols - test_cols\n",
    "    only_test = test_cols - train_cols\n",
    "    \n",
    "    if only_train:\n",
    "        print(f\"\\n仅在训练集中的特征({len(only_train)}个):\")\n",
    "        for col in sorted(only_train):\n",
    "            print(f\"  - {col}\")\n",
    "    \n",
    "    if only_test:\n",
    "        print(f\"\\n仅在测试集中的特征({len(only_test)}个):\")\n",
    "        for col in sorted(only_test):\n",
    "            print(f\"  - {col}\")\n",
    "    \n",
    "    # 对齐特征\n",
    "    print(\"\\n正在对齐特征...\")\n",
    "    common_cols = sorted(list(train_cols & test_cols))\n",
    "    \n",
    "    # 为训练集添加缺失特征\n",
    "    for col in only_test:\n",
    "        if col != 'CUST_NO':\n",
    "            train_mb_trnflw_features[col] = 0\n",
    "    \n",
    "    # 为测试集添加缺失特征\n",
    "    for col in only_train:\n",
    "        if col != 'CUST_NO':\n",
    "            test_mb_trnflw_features[col] = 0\n",
    "    \n",
    "    # 按照相同顺序排列列\n",
    "    all_cols = ['CUST_NO'] + sorted([col for col in train_mb_trnflw_features.columns if col != 'CUST_NO'])\n",
    "    train_mb_trnflw_features = train_mb_trnflw_features[all_cols]\n",
    "    test_mb_trnflw_features = test_mb_trnflw_features[all_cols]\n",
    "    \n",
    "    print(\"特征对齐完成!\")\n",
    "    print(f\"对齐后训练集特征数: {len(train_mb_trnflw_features.columns) - 1}\")\n",
    "    print(f\"对齐后测试集特征数: {len(test_mb_trnflw_features.columns) - 1}\")\n",
    "\n",
    "print(\"\\n\" + \"=\" * 80)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dd4b618",
   "metadata": {},
   "source": [
    "### 特征质量检查"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "fae82560",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "训练集特征质量检查\n",
      "================================================================================\n",
      "样本数: 1291\n",
      "特征数: 368\n",
      "\n",
      "缺失值统计:\n",
      "无缺失值\n",
      "\n",
      "无穷值检查:\n",
      "无无穷值\n",
      "\n",
      "常数特征检查:\n",
      "常数特征(24个): ['mb_trnflw_active_months', 'mb_trnflw_amt_bsn_combinations', 'mb_trnflw_daily_freq_outlier', 'mb_trnflw_interval_cv', 'mb_trnflw_interval_std', 'mb_trnflw_max_consecutive_days', 'mb_trnflw_month_start_amt_mean', 'mb_trnflw_month_start_amt_sum', 'mb_trnflw_month_start_count', 'mb_trnflw_month_start_ratio', 'mb_trnflw_monthly_amt_mean_std', 'mb_trnflw_monthly_amt_sum_std', 'mb_trnflw_monthly_count_std', 'mb_trnflw_outlier_low_count', 'mb_trnflw_paycod_7_amt_max', 'mb_trnflw_paycod_7_amt_mean', 'mb_trnflw_paycod_7_amt_sum', 'mb_trnflw_recency_score', 'mb_trnflw_weekday2_amt_mean', 'mb_trnflw_weekday2_amt_sum', 'mb_trnflw_weekday2_count', 'mb_trnflw_weekday3_amt_mean', 'mb_trnflw_weekday3_amt_sum', 'mb_trnflw_weekday3_count']\n",
      "\n",
      "================================================================================\n",
      "测试集特征质量检查\n",
      "================================================================================\n",
      "样本数: 457\n",
      "特征数: 368\n",
      "\n",
      "缺失值统计:\n",
      "无缺失值\n",
      "\n",
      "无穷值检查:\n",
      "无无穷值\n",
      "\n",
      "常数特征检查:\n",
      "常数特征(18个): ['mb_trnflw_amt_bsn_combinations', 'mb_trnflw_bsncod_15_amt_max', 'mb_trnflw_bsncod_15_amt_mean', 'mb_trnflw_bsncod_15_amt_sum', 'mb_trnflw_csttype_nunique', 'mb_trnflw_daily_freq_outlier', 'mb_trnflw_outlier_high_count', 'mb_trnflw_outlier_low_count', 'mb_trnflw_outlier_ratio', 'mb_trnflw_outlier_total_count', 'mb_trnflw_paycod_7_amt_max', 'mb_trnflw_paycod_7_amt_mean', 'mb_trnflw_paycod_7_amt_sum', 'mb_trnflw_paycod_7_count', 'mb_trnflw_paycod_8_amt_max', 'mb_trnflw_paycod_8_amt_mean', 'mb_trnflw_paycod_8_amt_sum', 'mb_trnflw_paycod_8_count']\n",
      "\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "print(\"=\" * 80)\n",
    "print(\"训练集特征质量检查\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "print(f\"样本数: {len(train_mb_trnflw_features)}\")\n",
    "print(f\"特征数: {len(train_mb_trnflw_features.columns) - 1}\")\n",
    "\n",
    "# 缺失值检查\n",
    "print(\"\\n缺失值统计:\")\n",
    "missing_train = train_mb_trnflw_features.isnull().sum()\n",
    "if missing_train.sum() > 0:\n",
    "    print(missing_train[missing_train > 0])\n",
    "else:\n",
    "    print(\"无缺失值\")\n",
    "\n",
    "# 无穷值检查\n",
    "print(\"\\n无穷值检查:\")\n",
    "inf_cols_train = []\n",
    "for col in train_mb_trnflw_features.columns:\n",
    "    if col != 'CUST_NO':\n",
    "        if np.isinf(train_mb_trnflw_features[col]).any():\n",
    "            inf_cols_train.append(col)\n",
    "if inf_cols_train:\n",
    "    print(f\"包含无穷值的特征({len(inf_cols_train)}个): {inf_cols_train[:10]}\")\n",
    "else:\n",
    "    print(\"无无穷值\")\n",
    "\n",
    "# 常数特征检查\n",
    "print(\"\\n常数特征检查:\")\n",
    "constant_cols_train = []\n",
    "for col in train_mb_trnflw_features.columns:\n",
    "    if col != 'CUST_NO':\n",
    "        if train_mb_trnflw_features[col].nunique() == 1:\n",
    "            constant_cols_train.append(col)\n",
    "if constant_cols_train:\n",
    "    print(f\"常数特征({len(constant_cols_train)}个): {constant_cols_train}\")\n",
    "else:\n",
    "    print(\"无常数特征\")\n",
    "\n",
    "print(\"\\n\" + \"=\" * 80)\n",
    "print(\"测试集特征质量检查\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "print(f\"样本数: {len(test_mb_trnflw_features)}\")\n",
    "print(f\"特征数: {len(test_mb_trnflw_features.columns) - 1}\")\n",
    "\n",
    "# 缺失值检查\n",
    "print(\"\\n缺失值统计:\")\n",
    "missing_test = test_mb_trnflw_features.isnull().sum()\n",
    "if missing_test.sum() > 0:\n",
    "    print(missing_test[missing_test > 0])\n",
    "else:\n",
    "    print(\"无缺失值\")\n",
    "\n",
    "# 无穷值检查\n",
    "print(\"\\n无穷值检查:\")\n",
    "inf_cols_test = []\n",
    "for col in test_mb_trnflw_features.columns:\n",
    "    if col != 'CUST_NO':\n",
    "        if np.isinf(test_mb_trnflw_features[col]).any():\n",
    "            inf_cols_test.append(col)\n",
    "if inf_cols_test:\n",
    "    print(f\"包含无穷值的特征({len(inf_cols_test)}个): {inf_cols_test[:10]}\")\n",
    "else:\n",
    "    print(\"无无穷值\")\n",
    "\n",
    "# 常数特征检查\n",
    "print(\"\\n常数特征检查:\")\n",
    "constant_cols_test = []\n",
    "for col in test_mb_trnflw_features.columns:\n",
    "    if col != 'CUST_NO':\n",
    "        if test_mb_trnflw_features[col].nunique() == 1:\n",
    "            constant_cols_test.append(col)\n",
    "if constant_cols_test:\n",
    "    print(f\"常数特征({len(constant_cols_test)}个): {constant_cols_test}\")\n",
    "else:\n",
    "    print(\"无常数特征\")\n",
    "\n",
    "print(\"\\n\" + \"=\" * 80)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "476d43de",
   "metadata": {},
   "source": [
    "### 特征统计描述"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "04b37ebc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练集特征描述性统计(前20个特征):\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_days_last14d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>1.014717</td>\n",
       "      <td>0.120465</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_days_last1d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.967467</td>\n",
       "      <td>0.177479</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_days_last30d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>1.014717</td>\n",
       "      <td>0.120465</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_days_last3d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.967467</td>\n",
       "      <td>0.177479</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_days_last60d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>1.014717</td>\n",
       "      <td>0.120465</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_days_last7d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>1.014717</td>\n",
       "      <td>0.120465</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_days_last90d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>1.014717</td>\n",
       "      <td>0.120465</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_months</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_rate</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.498528</td>\n",
       "      <td>0.012047</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_rate_last14d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.072480</td>\n",
       "      <td>0.008605</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_rate_last1d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.967467</td>\n",
       "      <td>0.177479</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_rate_last30d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.033824</td>\n",
       "      <td>0.004016</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.066667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_rate_last3d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.322489</td>\n",
       "      <td>0.059160</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_rate_last60d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.016912</td>\n",
       "      <td>0.002008</td>\n",
       "      <td>0.016667</td>\n",
       "      <td>0.016667</td>\n",
       "      <td>0.016667</td>\n",
       "      <td>0.016667</td>\n",
       "      <td>0.033333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_rate_last7d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.144960</td>\n",
       "      <td>0.017209</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_rate_last90d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.011275</td>\n",
       "      <td>0.001339</td>\n",
       "      <td>0.011111</td>\n",
       "      <td>0.011111</td>\n",
       "      <td>0.011111</td>\n",
       "      <td>0.011111</td>\n",
       "      <td>0.022222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_span</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>1.044152</td>\n",
       "      <td>0.361396</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_active_trend_7d_30d</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>4.284438</td>\n",
       "      <td>0.000077</td>\n",
       "      <td>4.284429</td>\n",
       "      <td>4.284429</td>\n",
       "      <td>4.284429</td>\n",
       "      <td>4.284429</td>\n",
       "      <td>4.285072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_amt_0_100_count</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>0.123935</td>\n",
       "      <td>0.482380</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mb_trnflw_amt_0_100_sum</th>\n",
       "      <td>1291.0</td>\n",
       "      <td>6.022881</td>\n",
       "      <td>26.610233</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>400.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                count      mean        std       min  \\\n",
       "mb_trnflw_active_days_last14d  1291.0  1.014717   0.120465  1.000000   \n",
       "mb_trnflw_active_days_last1d   1291.0  0.967467   0.177479  0.000000   \n",
       "mb_trnflw_active_days_last30d  1291.0  1.014717   0.120465  1.000000   \n",
       "mb_trnflw_active_days_last3d   1291.0  0.967467   0.177479  0.000000   \n",
       "mb_trnflw_active_days_last60d  1291.0  1.014717   0.120465  1.000000   \n",
       "mb_trnflw_active_days_last7d   1291.0  1.014717   0.120465  1.000000   \n",
       "mb_trnflw_active_days_last90d  1291.0  1.014717   0.120465  1.000000   \n",
       "mb_trnflw_active_months        1291.0  1.000000   0.000000  1.000000   \n",
       "mb_trnflw_active_rate          1291.0  0.498528   0.012047  0.400000   \n",
       "mb_trnflw_active_rate_last14d  1291.0  0.072480   0.008605  0.071429   \n",
       "mb_trnflw_active_rate_last1d   1291.0  0.967467   0.177479  0.000000   \n",
       "mb_trnflw_active_rate_last30d  1291.0  0.033824   0.004016  0.033333   \n",
       "mb_trnflw_active_rate_last3d   1291.0  0.322489   0.059160  0.000000   \n",
       "mb_trnflw_active_rate_last60d  1291.0  0.016912   0.002008  0.016667   \n",
       "mb_trnflw_active_rate_last7d   1291.0  0.144960   0.017209  0.142857   \n",
       "mb_trnflw_active_rate_last90d  1291.0  0.011275   0.001339  0.011111   \n",
       "mb_trnflw_active_span          1291.0  1.044152   0.361396  1.000000   \n",
       "mb_trnflw_active_trend_7d_30d  1291.0  4.284438   0.000077  4.284429   \n",
       "mb_trnflw_amt_0_100_count      1291.0  0.123935   0.482380  0.000000   \n",
       "mb_trnflw_amt_0_100_sum        1291.0  6.022881  26.610233  0.000000   \n",
       "\n",
       "                                    25%       50%       75%         max  \n",
       "mb_trnflw_active_days_last14d  1.000000  1.000000  1.000000    2.000000  \n",
       "mb_trnflw_active_days_last1d   1.000000  1.000000  1.000000    1.000000  \n",
       "mb_trnflw_active_days_last30d  1.000000  1.000000  1.000000    2.000000  \n",
       "mb_trnflw_active_days_last3d   1.000000  1.000000  1.000000    1.000000  \n",
       "mb_trnflw_active_days_last60d  1.000000  1.000000  1.000000    2.000000  \n",
       "mb_trnflw_active_days_last7d   1.000000  1.000000  1.000000    2.000000  \n",
       "mb_trnflw_active_days_last90d  1.000000  1.000000  1.000000    2.000000  \n",
       "mb_trnflw_active_months        1.000000  1.000000  1.000000    1.000000  \n",
       "mb_trnflw_active_rate          0.500000  0.500000  0.500000    0.500000  \n",
       "mb_trnflw_active_rate_last14d  0.071429  0.071429  0.071429    0.142857  \n",
       "mb_trnflw_active_rate_last1d   1.000000  1.000000  1.000000    1.000000  \n",
       "mb_trnflw_active_rate_last30d  0.033333  0.033333  0.033333    0.066667  \n",
       "mb_trnflw_active_rate_last3d   0.333333  0.333333  0.333333    0.333333  \n",
       "mb_trnflw_active_rate_last60d  0.016667  0.016667  0.016667    0.033333  \n",
       "mb_trnflw_active_rate_last7d   0.142857  0.142857  0.142857    0.285714  \n",
       "mb_trnflw_active_rate_last90d  0.011111  0.011111  0.011111    0.022222  \n",
       "mb_trnflw_active_span          1.000000  1.000000  1.000000    4.000000  \n",
       "mb_trnflw_active_trend_7d_30d  4.284429  4.284429  4.284429    4.285072  \n",
       "mb_trnflw_amt_0_100_count      0.000000  0.000000  0.000000    8.000000  \n",
       "mb_trnflw_amt_0_100_sum        0.000000  0.000000  0.000000  400.000000  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_cols = [col for col in train_mb_trnflw_features.columns if col != 'CUST_NO']\n",
    "\n",
    "print(\"训练集特征描述性统计(前20个特征):\")\n",
    "train_mb_trnflw_features[feature_cols[:20]].describe().T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "338572d6",
   "metadata": {},
   "source": [
    "### 保存特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f7d0538",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_save_path = './feature'\n",
    "os.makedirs(feature_save_path, exist_ok=True)\n",
    "\n",
    "train_feature_file = os.path.join(feature_save_path, 'train_mb_trnflw_features.pkl')\n",
    "test_feature_file = os.path.join(feature_save_path, 'test_mb_trnflw_features.pkl')\n",
    "\n",
    "print(\"正在保存特征...\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "with open(train_feature_file, 'wb') as f:\n",
    "    pickle.dump(train_mb_trnflw_features, f)\n",
    "print(f\"训练集特征已保存至: {train_feature_file}\")\n",
    "print(f\"文件大小: {os.path.getsize(train_feature_file) / 1024 / 1024:.2f} MB\")\n",
    "\n",
    "with open(test_feature_file, 'wb') as f:\n",
    "    pickle.dump(test_mb_trnflw_features, f)\n",
    "print(f\"测试集特征已保存至: {test_feature_file}\")\n",
    "print(f\"文件大小: {os.path.getsize(test_feature_file) / 1024 / 1024:.2f} MB\")\n",
    "\n",
    "print(\"\\n\" + \"=\" * 80)\n",
    "print(\"掌银金融性交易流水表特征工程完成!\")\n",
    "print(\"=\" * 80)\n",
    "print(f\"训练集: {train_mb_trnflw_features.shape}\")\n",
    "print(f\"测试集: {test_mb_trnflw_features.shape}\")\n",
    "print(f\"总特征数: {len(train_mb_trnflw_features.columns) - 1}\")\n",
    "print(\"=\" * 80)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f47bcb33",
   "metadata": {},
   "source": [
    "### 特征验证与加载测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42f72cf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"特征加载测试...\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "# 重新加载特征\n",
    "with open(train_feature_file, 'rb') as f:\n",
    "    loaded_train_features = pickle.load(f)\n",
    "\n",
    "with open(test_feature_file, 'rb') as f:\n",
    "    loaded_test_features = pickle.load(f)\n",
    "\n",
    "print(f\"训练集特征加载成功: {loaded_train_features.shape}\")\n",
    "print(f\"测试集特征加载成功: {loaded_test_features.shape}\")\n",
    "\n",
    "# 验证数据一致性\n",
    "assert loaded_train_features.shape == train_mb_trnflw_features.shape, \"训练集形状不一致\"\n",
    "assert loaded_test_features.shape == test_mb_trnflw_features.shape, \"测试集形状不一致\"\n",
    "assert list(loaded_train_features.columns) == list(train_mb_trnflw_features.columns), \"训练集列名不一致\"\n",
    "assert list(loaded_test_features.columns) == list(test_mb_trnflw_features.columns), \"测试集列名不一致\"\n",
    "\n",
    "print(\"\\n特征验证通过!\")\n",
    "print(\"=\" * 80)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a256fd39",
   "metadata": {},
   "source": [
    "### 特征重要性分析(可选)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfeb50ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"特征概览:\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "feature_groups = {\n",
    "    '基础统计特征': [col for col in feature_cols if any(x in col for x in ['_count', '_amt_sum', '_amt_mean', '_amt_median', '_amt_std', '_amt_min', '_amt_max', '_amt_q'])],\n",
    "    '时间窗口特征': [col for col in feature_cols if 'last' in col and 'd' in col],\n",
    "    '周末工作日特征': [col for col in feature_cols if 'weekend' in col or 'weekday' in col],\n",
    "    '业务代码特征': [col for col in feature_cols if 'bsncod' in col],\n",
    "    '状态特征': [col for col in feature_cols if '_stt_' in col or '_stt' == col[-4:]],\n",
    "    '代收付类型特征': [col for col in feature_cols if 'paycod' in col],\n",
    "    '客户类别特征': [col for col in feature_cols if 'csttype' in col],\n",
    "    '金额分档特征': [col for col in feature_cols if any(x in col for x in ['_amt_0_', '_amt_100_', '_amt_500_', '_amt_1k_', '_amt_5k_', '_amt_10k_', '_amt_50k_', '_amt_100k_', 'large_amt', 'small_amt'])],\n",
    "    '时间周期特征': [col for col in feature_cols if any(x in col for x in ['monthly', 'weekly', 'month_start', 'month_end', 'mid_month'])],\n",
    "    '连续性特征': [col for col in feature_cols if any(x in col for x in ['consecutive', 'gap', 'interval'])],\n",
    "    '高级统计特征': [col for col in feature_cols if any(x in col for x in ['_skew', '_kurt', 'zero_amt', 'dup_amt'])],\n",
    "    '交叉特征': [col for col in feature_cols if 'combinations' in col or 'cross' in col],\n",
    "    'RFM特征': [col for col in feature_cols if any(x in col for x in ['recency', 'frequency', 'monetary', 'rfm'])],\n",
    "    '异常特征': [col for col in feature_cols if 'outlier' in col],\n",
    "    '时序变化特征': [col for col in feature_cols if any(x in col for x in ['trend', 'recent', 'volatility', 'first_last'])],\n",
    "    '其他特征': []\n",
    "}\n",
    "\n",
    "# 找出未分类的特征\n",
    "classified_features = set()\n",
    "for group_features in feature_groups.values():\n",
    "    classified_features.update(group_features)\n",
    "\n",
    "feature_groups['其他特征'] = [col for col in feature_cols if col not in classified_features]\n",
    "\n",
    "# 打印各组特征数量\n",
    "for group_name, group_features in feature_groups.items():\n",
    "    if len(group_features) > 0:\n",
    "        print(f\"{group_name}: {len(group_features)}个\")\n",
    "\n",
    "print(\"\\n\" + \"=\" * 80)\n",
    "print(f\"总计: {len(feature_cols)}个特征\")\n",
    "print(\"=\" * 80)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "starcup",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
